Packages and utilities

library(tidyverse)
library(lme4)
library(lmerTest)
library(logging)
library(mvtnorm)
library(mgcv)
# Compute the log-likelihood of a new dataset using a fit lme4 model.
logLik_test <- function(lm, test_X, test_y) {
  predictions <- predict(lm, test_X, re.form=NA)
  # Get std.dev. of residual, estimated from train data
  stdev <- sigma(lm)
  # For each prediction--observation, get the density p(obs | N(predicted, model_sigma)) and reduce
  density <- sum(dnorm(test_y, predictions, stdev, log=TRUE))
  return(density)
}
# Get per-prediction log-likelihood
logLik_test_per <- function(lm, test_X, test_y) {
  predictions <- predict(lm, test_X, re.form=NA)
  # Get std.dev. of residual, estimated from train data
  stdev <- sigma(lm)
  # For each prediction--observation, get the density p(obs | N(predicted, model_sigma))
  densities <- dnorm(test_y, predictions, stdev, log=TRUE)
  return(densities)
}
# Compute MSE of a new dataset using a fit lme4 model.
mse_test <- function(lm, test_X, test_y) {
  return(mean((predict(lm, test_X, re.form=NA) - test_y) ^ 2))
}
#Sanity checks
#mylm <- gam(psychometric ~  s(surprisal, bs = "cr", k = 20) + s(prev_surp, bs = "cr", k = 20) + te(freq, len, bs = "cr") + te(prev_freq, prev_len, bs = "cr"), data=train_data)
#c(logLik(mylm), logLik_test(mylm, train_data, train_data$psychometric))
#logLik_test(mylm, test_data, test_data$psychometric)

Data loading and preprocessing

data = read.csv("../data/harmonized_results.csv")

all_data = data %>%
  mutate(seed = as.factor(seed)) %>%
  group_by(corpus, model, training, seed) %>%
    mutate(prev_surp = lag(surprisal),
         prev_code = lag(code),
         prev_len = lag(len),
         prev_freq = lag(freq),
         prev_surp = lag(surprisal),
         
         prev2_freq = lag(prev_freq),
         prev2_code = lag(prev_code),
         prev2_len = lag(prev_len),
         prev2_surp = lag(prev_surp),
         
         prev3_freq = lag(prev2_freq),
         prev3_code = lag(prev2_code),
         prev3_len = lag(prev2_len),
         prev3_surp = lag(prev2_surp)) %>%
  ungroup() %>%
  
  # Filter back three for the dundee corpus. Filter back 1 for all other corpora
  # NB this effectively removes all zero-surprisal rows, since early-sentence tokens don't have contiguous token history
  filter((corpus == "dundee" & code == prev2_code + 2) | (corpus != "dundee" & code == prev_code + 1)) %>%
  
  select(-prev_code, -prev2_code, -prev3_code) %>%
  drop_na()

all_data = all_data %>%
  mutate(
    model = as.character(model),
    model = if_else(model == "gpt-2", "gpt2", model),
    model = as.factor(model))
missing_rows = all_data %>% complete(nesting(corpus, code), nesting(model, training, seed)) %>% 
  group_by(corpus, code) %>% 
    filter(sum(is.na(surprisal)) > 0) %>% 
  ungroup() %>% 
  anti_join(all_data, by=c("corpus", "code", "model", "training", "seed"))

missing_rows %>% ggplot(aes(x=corpus, fill=factor(paste(model,training)))) + geom_bar(position=position_dodge(width=0.8))

print(missing_rows %>% group_by(model, training, seed, corpus) %>% summarise(n=n())) %>% arrange(desc(n))

# TODO why are there 3737 missing rows for gptbpe at all corpus sizes on natural-stories? they only have 1 missing row on Dundee

# Compute the ideal number of model--seed--training observations per token.
ideal_token_obs_count = all_data %>% group_by(corpus, code) %>% group_size() %>% max()

to_drop = all_data %>% group_by(corpus, code) %>% filter(n() != ideal_token_obs_count) %>% ungroup()
loginfo(paste("Dropping", nrow(to_drop), "observations corresponding to corpus tokens which are missing observations for some model."))
2020-05-08 11:29:50 INFO::Dropping 123604 observations corresponding to corpus tokens which are missing observations for some model.
loginfo(paste("Dropping", to_drop %>% group_by(corpus, code) %>% n_groups(), "tokens which are missing observations for some model."))
2020-05-08 11:29:50 INFO::Dropping 8834 tokens which are missing observations for some model.
all_data = all_data %>% anti_join(to_drop %>% group_by(corpus, code), by=c("corpus", "code"))
loginfo(paste("After drop,", nrow(all_data), "observations (", all_data %>% group_by(corpus, code) %>% n_groups(), " tokens) remain."))
2020-05-08 11:29:51 INFO::After drop, 999775 observations ( 34475  tokens) remain.

to_drop_zeros = all_data %>% group_by(corpus, code) %>% filter(any(surprisal == 0)) %>% ungroup()
loginfo(paste("Dropping", nrow(to_drop_zeros), "observations corresponding to corpus tokens which have surprisal zeros for some model."))
2020-05-08 11:29:51 INFO::Dropping 116 observations corresponding to corpus tokens which have surprisal zeros for some model.
loginfo(paste("Dropping", to_drop_zeros %>% group_by(corpus, code) %>% n_groups(), "tokens which have surprisal zeros for some model."))
2020-05-08 11:29:51 INFO::Dropping 4 tokens which have surprisal zeros for some model.
all_data = all_data %>% anti_join(to_drop_zeros %>% group_by(corpus, code), by=c("corpus", "code"))
loginfo(paste("After drop,", nrow(all_data), "observations (", all_data %>% group_by(corpus, code) %>% n_groups(), " tokens) remain."))
2020-05-08 11:29:52 INFO::After drop, 999659 observations ( 34471  tokens) remain.

Learn models

# Compute linear model stats for the given training data subset and full test data.
# Automatically subsets the test data to match the relevant group for which we are training a linear model.
get_lm_data <- function(df, test_data, formula, fold, store_env) {
  #this_lm <- gam(formula, data=df);
  this_lm = lm(formula, data=df)
  this_test_data <- semi_join(test_data, df, by=c("training", "model", "seed", "corpus"));
  
  # Save lm to the global env so that we can access residuals later.
  lm_name = paste(unique(paste(df$model, df$training, df$seed, df$corpus))[1], fold)
  assign(lm_name, this_lm, envir=store_env)
  
  summarise(df,
            log_lik = as.numeric(logLik(this_lm, REML = F)),
            test_lik = logLik_test(this_lm, this_test_data, this_test_data$psychometric),
            test_mse = mse_test(this_lm, this_test_data, this_test_data$psychometric))
}
# For a previously fitted lm stored in store_env, get the residuals on test data of the relevant data subset.
get_lm_residuals <- function(df, fold, store_env) {
  # Retrieve the relevant lm.
  lm_name = paste(unique(paste(df$model, df$training, df$seed, df$corpus))[1], fold)
  this_lm <- get(lm_name, envir=store_env)
  
  mutate(df,
         likelihood = logLik_test_per(this_lm, df, df$psychometric),
         resid = df$psychometric - predict(this_lm, df, re.form=NA))
}
# Compute per-example delta-log-likelihood for the given test fold.
get_lm_delta_log_lik <- function(test_data, fold, baseline_env, full_env) {
  lm_name = paste(unique(paste(test_data$model, test_data$training, test_data$seed, test_data$corpus))[1], fold)
  baseline_lm <- get(lm_name, envir=baseline_env)
  full_lm <- get(lm_name, envir=full_env)
  
  delta_log_lik = logLik_test_per(full_lm, test_data, test_data$psychometric) - logLik_test_per(baseline_lm, test_data, test_data$psychometric)
  return(cbind(test_data, delta_log_lik=delta_log_lik))
}
#####
# Define regression formulae.
# Eye-tracking regression: only use surprisal and previous surprisal; SPRT regression: use 2-back features.
#baseline_rt_regression = psychometric ~ te(freq, len, bs = "cr") + te(prev_freq, prev_len, bs = "cr")
#baselie_sprt_regression = psychometric ~ te(freq, len, bs = "cr") + te(prev_freq, prev_len, bs = "cr") + te(prev2_freq, prev2_len, bs = "cr")
#full_rt_regression = (psychometric ~ s(surprisal, bs = "cr", k = 20) + s(prev_surp, bs = "cr", k = 20)
                     #+ te(freq, len, bs = "cr") + te(prev_freq, prev_len, bs = "cr"))
#full_sprt_regression = (psychometric ~ s(surprisal, bs = "cr", k = 20) + s(prev_surp, bs = "cr", k = 20) + s(prev2_surp, bs = "cr", k = 20)
                        #+ te(freq, len, bs = "cr") + te(prev_freq, prev_len, bs = "cr") + te(prev2_freq, prev2_len, bs = "cr"))

baseline_rt_regression = psychometric ~ freq + prev_freq + prev2_freq + prev3_freq + len + prev_len + prev2_len + prev3_len
baseline_sprt_regression = psychometric ~ freq + prev_freq + len + prev_len

full_rt_regression = psychometric ~ surprisal + prev_surp + prev2_surp + prev3_surp + freq + prev_freq + prev2_freq + prev3_freq + len + prev_len + prev2_len + prev3_len
full_sprt_regression = psychometric ~ surprisal + prev_surp + freq + prev_freq + len + prev_len
  
#####
# Prepare frames/environments for storing results/objects.
baseline_results = data.frame()
full_model_results = data.frame()
baseline_residuals = data.frame()
full_residuals = data.frame()
log_lik_deltas = data.frame()

#Randomly shuffle the data
all_data<-all_data[sample(nrow(all_data)),]
#Create K equally size folds
K = 5
folds <- cut(seq(1,nrow(all_data)),breaks=K,labels=FALSE)
#Perform 10 fold cross validation

# Fit models for some fold of the data.
baseline_corpus = function(corpus, df, test_data, fold, env) {
  if(corpus == "dundee") {
    get_lm_data(df, test_data, baseline_rt_regression, fold, env)
  } else {
    get_lm_data(df, test_data, baseline_sprt_regression, fold, env)
  }
}
full_model_corpus = function(corpus, df, test_data, fold, env) {
  if(corpus[1] == "dundee") {
    get_lm_data(df, test_data, full_rt_regression, fold, env)
  } else {
    get_lm_data(df, test_data, full_sprt_regression, fold, env)
  }
}

# Prepare a new Environment in which we store fitted LMs, which we'll query later for residuals and other metrics.
baseline_env = new.env()
full_env = new.env()

for(i in 1:K) { 
  #Segement your data by fold using the which() function 
  testIndexes <- which(folds==i, arr.ind=TRUE)
  test_data <- all_data[testIndexes, ]
  train_data <- all_data[-testIndexes, ]
  
  # Compute a baseline linear model for each model--training--seed--RT-corpus combination.
  baselines = train_data %>%
    group_by(model, training, seed, corpus) %>%
      print(model) %>%
      do(baseline_corpus(unique(.$corpus), ., test_data, i, baseline_env)) %>%
    ungroup() %>%
    mutate(seed = as.factor(seed),
           fold = i)
  
  baseline_results = rbind(baseline_results, baselines)
  
  # Compute a full linear model for each model--training--seed-RT-corpus combination
  full_models = train_data %>%
    group_by(model, training, seed, corpus) %>%
      do(full_model_corpus(unique(.$corpus), ., test_data, i, full_env)) %>%
    ungroup() %>%
    mutate(seed = as.factor(seed),
           fold = i)
  
  full_model_results = rbind(full_model_results, full_models)
  
  # Compute delta-log-likelihoods
  fold_log_lik_deltas = test_data %>%
    group_by(model, training, seed, corpus) %>%
      do(get_lm_delta_log_lik(., i, baseline_env, full_env)) %>%
    ungroup()

  log_lik_deltas = rbind(log_lik_deltas, fold_log_lik_deltas)
  
  fold_baseline_residuals = test_data %>%
    group_by(model, training, seed, corpus) %>%
      do(get_lm_residuals(., i, baseline_env)) %>%
    ungroup()

  baseline_residuals = rbind(baseline_residuals, fold_baseline_residuals)

  fold_full_residuals = test_data %>%
    group_by(model, training, seed, corpus) %>%
      do(get_lm_residuals(., i, full_env)) %>%
    ungroup()

  full_residuals = rbind(full_residuals, fold_full_residuals)
}

|===========================================================================================================================================================                         | 86% ~0 s remaining     
|==============================================================================================================================================================                      | 88% ~0 s remaining     
|====================================================================================================================================================================                | 91% ~0 s remaining     
|==========================================================================================================================================================================          | 95% ~0 s remaining     
|================================================================================================================================================================================    | 98% ~0 s remaining     
|=================================================================================================================================================                                   | 81% ~0 s remaining     
|========================================================================================================================================================                            | 84% ~0 s remaining     
|==============================================================================================================================================================                      | 88% ~0 s remaining     
|====================================================================================================================================================================                | 91% ~0 s remaining     
|==========================================================================================================================================================================          | 95% ~0 s remaining     
|================================================================================================================================================================================    | 98% ~0 s remaining     

|====================================================================================================================================================================                | 91% ~0 s remaining     
|==========================================================================================================================================================================          | 95% ~0 s remaining     
|=============================================================================================================================================================================       | 97% ~0 s remaining     
|================================================================================================================================================================================    | 98% ~0 s remaining     
|========================================================================================================================================================                            | 84% ~0 s remaining     
|==============================================================================================================================================================                      | 88% ~0 s remaining     
|====================================================================================================================================================================                | 91% ~0 s remaining     
|==========================================================================================================================================================================          | 95% ~0 s remaining     
|================================================================================================================================================================================    | 98% ~0 s remaining     

|=====================================================================================================================                                                               | 66% ~1 s remaining     
|=========================================================================================================================                                                           | 67% ~1 s remaining     
|===============================================================================================================================                                                     | 71% ~1 s remaining     
|=====================================================================================================================================                                               | 74% ~1 s remaining     
|===========================================================================================================================================                                         | 78% ~1 s remaining     
|=================================================================================================================================================                                   | 81% ~1 s remaining     
|========================================================================================================================================================                            | 84% ~0 s remaining     
|==============================================================================================================================================================                      | 88% ~0 s remaining     
|====================================================================================================================================================================                | 91% ~0 s remaining     
|==========================================================================================================================================================================          | 95% ~0 s remaining     
|================================================================================================================================================================================    | 98% ~0 s remaining     
|========================================================================================================================================================                            | 84% ~0 s remaining     
|==============================================================================================================================================================                      | 88% ~0 s remaining     
|====================================================================================================================================================================                | 91% ~0 s remaining     
|==========================================================================================================================================================================          | 95% ~0 s remaining     
|================================================================================================================================================================================    | 98% ~0 s remaining     

|====================================================================================================================================================================                | 91% ~0 s remaining     
|==========================================================================================================================================================================          | 95% ~0 s remaining     
|================================================================================================================================================================================    | 98% ~0 s remaining     
|==============================================================================================================================================================                      | 88% ~0 s remaining     
|====================================================================================================================================================================                | 91% ~0 s remaining     
|==========================================================================================================================================================================          | 95% ~0 s remaining     
|================================================================================================================================================================================    | 98% ~0 s remaining     

|=================================================================================================================================================================                   | 90% ~0 s remaining     
|====================================================================================================================================================================                | 91% ~0 s remaining     
|==========================================================================================================================================================================          | 95% ~0 s remaining     
|================================================================================================================================================================================    | 98% ~0 s remaining     
|===========================================================================================================================================================                         | 86% ~0 s remaining     
|==============================================================================================================================================================                      | 88% ~0 s remaining     
|====================================================================================================================================================================                | 91% ~0 s remaining     
|==========================================================================================================================================================================          | 95% ~0 s remaining     
|================================================================================================================================================================================    | 98% ~0 s remaining     
#write.csv(full_residuals, "../data/analysis_checkpoints/full_residuals.csv")
#write.csv(baseline_residuals, "../data/analysis_checkpoints/baseline_residuals.csv")
model_deltas = log_lik_deltas %>%
  group_by(model, training, seed, corpus) %>% 
  summarise(mean_delta_log_lik = mean(delta_log_lik),
            sem_delta_log_lik = sd(delta_log_lik) / sqrt(length(delta_log_lik)))
write.csv(full_model_results, "../data/analysis_checkpoints/full_model_result.csv")
write.csv(baseline_results, "../data/analysis_checkpoints/baseline_results.csv")
#full_model_results = read.csv("../data/analysis_checkpoints/ffull_model_results.csv")
#baseline_results = read.csv("../data/analysis_checkpoints/fbaseline_resultsb.csv")
metric <- "ΔLogLik"
#metric <- "-ΔMSE"

# # Select the relevant metric.
model_deltas = model_deltas %>%
    # Retrieve the current test metric
    mutate(delta_test_mean = mean_delta_log_lik,
           delta_test_sem = sem_delta_log_lik) %>%
    # mutate(delta_test_mean = mean_delta_mse,
    #        delta_test_sem = sem_delta_mse)
    
    # Remove the raw metrics.
    select(-mean_delta_log_lik, -sem_delta_log_lik,
           #-mean_delta_mse, -sem_delta_mse
           )
model_deltas
# Sanity check: training on train+test data should yield improved performance over training on just training data. (When evaluating on test data.)
# full_baselines = all_data %>%
#   group_by(model, training, seed, corpus) %>%
#   summarise(baseline_train_all_test_lik = logLik_test(lm(psychometric ~ len + freq + sent_pos, data=.), semi_join(test_data, ., by=c("training", "model", "seed", "corpus")), semi_join(test_data, ., by=c("training", "model", "seed", "corpus"))$psychometric)) %>%
#   ungroup()
# full_baselines
# 
# full_baselines %>%
#   right_join(baselines, by=c("seed", "training", "model", "corpus")) %>%
#   mutate(delta=baseline_train_all_test_lik-baseline_test_lik) %>%
#   select(-baseline_lik) # %>%
#   #select(-baseline_test_lik, -baseline_train_all_test_lik, -baseline_lik, -baseline_test_mse)

Load language model data (SyntaxGym, PPL)

language_model_data = read.csv("../data/model_metadata.csv") %>%
  mutate(model = as.character(model),
         model = if_else(model == "gpt-2", "gpt2", model),
         model = as.factor(model)) %>%
  mutate(train_size = case_when(str_starts(training, "bllip-lg") ~ 42,
                                str_starts(training, "bllip-md") ~ 15,
                                str_starts(training, "bllip-sm") ~ 5,
                                str_starts(training, "bllip-xs") ~ 1),
         
         # Training vocabulary usually covaries with the training corpus.
         # But BPE models share a vocabulary across training corpora.
         training_vocab=as.factor(ifelse(str_detect(training, "gptbpe"), "gptbpe", as.character(training)))
         ) %>%
  mutate(seed = as.factor(seed)) %>%
  select(-pid, -test_loss) %>%
  distinct(model, training, seed, .keep_all = TRUE)
table(language_model_data$seed)

         0        111        120        922       1111       3602       4301       7245       7877      28066      28068      44862      51272      64924 1581807512 1581807578 1581861474 1581955288 
         4          7          6          5          4          1          1          1          1          1          1          1          1          1          1          1          1          1 
1582126320 1586986276 1587139950 
         1          1          1 
table(model_deltas$seed)

       111        120        922       1111       3602       4301       7245       7877      28066      28068      44862      51272      64924 1581807512 1581807578 1581861474 1581955288 1582126320 
         6          6          6          8          2          2          2          2          2          2          2          2          2          2          2          2          2          2 
1586986276 1587139950 
         2          2 

First join delta-metric data with model auxiliary data.

model_deltas = model_deltas %>%
  merge(language_model_data, by = c("seed", "training", "model"), all=T) %>%
  drop_na()

model_deltas

Also join on the original linear model data, rather than collapsing to delta-metrics. This will support regressions later on that don’t collapse across folds.

Final data preprocessing

# Exclude ordered-neurons from all analyses.
model_deltas <- model_deltas %>%
  filter(model != "ordered-neurons")

Visualizations

The basics

all_data %>% ggplot(aes(x=corpus)) + geom_bar()

print(all_data %>% group_by(corpus) %>% summarise(n=n()))
all_data %>% 
  ggplot(aes(x=freq, color=corpus)) + geom_density()

all_data %>% 
  ggplot(aes(x=len, color=corpus)) + geom_density()

all_data %>% 
  ggplot(aes(x=surprisal, color=corpus)) + geom_density()

Predictive power and SG

model_deltas %>%
  ggplot(aes(x=sg_score, y=delta_test_mean)) +
    geom_errorbar(aes(ymin=delta_test_mean-delta_test_sem, ymax=delta_test_mean+delta_test_sem)) +
    geom_smooth(method="lm", se=T) +
    geom_point(stat="identity", position="dodge", alpha=1, size=3, aes(color=training_vocab, shape=model)) +
    ylab(metric) +
    xlab("Syntax Generalization Score") +
    ggtitle("Syntactic Generalization vs. Predictive Power") +
    scale_color_manual(values = c("bllip-lg"="#440154FF",
                              "bllip-md"="#39568CFF",
                              "bllip-sm"="#1F968BFF",
                              "bllip-xs"="#73D055FF",
                              "gptbpe"="#888888")) +
    facet_grid(~corpus, scales="free") +
    theme(axis.text=element_text(size=14),
          strip.text.x = element_text(size=14),
          legend.text=element_text(size=14),
          axis.title=element_text(size=18),
          legend.position = "bottom")

#ggsave("./cogsci_images/sg_loglik.png",height=5,width=6)

Regression analyses

We control for effects of perplexity by relating the residuals of a performance ~ PPL regression to SG score.

# Prepare a residualized regression for x1 onto y, controlling for the effects of x2.
d_resid = model_deltas %>%
  drop_na() %>%
  
  # Residualize delta metric w.r.t PPL for each model--training--seed
  group_by(corpus) %>%
    mutate(resid.delta = resid(lm(delta_test_mean ~ training:test_ppl))) %>%
  ungroup() %>%
  
  # Residualize SG score w.r.t. PPL for each training vocabulary
  group_by(training_vocab) %>%
    # NB no need for training:ppl interaction, since we're within-group.
    mutate(resid.sg = resid(lm(sg_score ~ test_ppl))) %>%
  ungroup() %>%
  # Compute summary statistics across model--training--seed--corpus.
  group_by(model, training_vocab, corpus, seed) %>%
    summarise(resid.delta.mean = mean(resid.delta),
              resid.delta.sem = sd(resid.delta) / sqrt(length(resid.delta)),
              resid.sg.mean = mean(resid.sg),
              resid.sg.sem = sd(resid.sg) / sqrt(length(resid.sg)))
# Now plot residual vs SG
d_resid %>%
  #filter(corpus != "bnc-brown") %>%
  ggplot(aes(x=resid.sg.mean, y=resid.delta.mean)) +
    geom_errorbar(aes(xmin=resid.sg.mean - resid.sg.sem,
                      xmax=resid.sg.mean + resid.sg.sem,
                      ymin=resid.delta.mean - resid.delta.sem,
                      ymax=resid.delta.mean + resid.delta.sem), alpha=0.3) +
    geom_smooth(method="lm", se=T) +
    geom_point(stat="identity", position="dodge", alpha=1, size=4, aes(shape=model, color=training_vocab)) +
    ylab(paste("Residual", metric)) +
    xlab("Residual Syntax Generalization Score") +
    ggtitle("Syntactic Generalization vs. Predictive Power") +
    scale_color_manual(values = c("bllip-lg"="#440154FF",
                                  "bllip-md"="#39568CFF",
                                  "bllip-sm"="#1F968BFF",
                                  "bllip-xs"="#73D055FF",
                                  "gptbpe"="#888888")) +
    facet_grid(.~corpus, scales="free") +
    theme(axis.text=element_text(size=14),
          strip.text.x = element_text(size=14),
          legend.text=element_text(size=14),
          axis.title=element_text(size=18),
          legend.position = "right")
Ignoring unknown aesthetics: xmin, xmax
ggsave("../images/cuny2020/ppl_sg.png",height=4.5,width=11)

do_stepwise_regression = function(cur_corpus) {
  regression_data = model_deltas %>%
    filter(corpus == cur_corpus)
  
  print("----------------------")
  print(cur_corpus)
  
  lm1 = lm(delta_test_mean ~ training_vocab:test_ppl, data = regression_data)
  lm2 = lm(delta_test_mean ~ training_vocab:test_ppl + sg_score, data = regression_data)
  print(anova(lm1, lm2))
  summary(lm2)
}
#do_stepwise_regression("bnc-brown")
do_stepwise_regression("dundee")
[1] "----------------------"
[1] "dundee"
Analysis of Variance Table

Model 1: delta_test_mean ~ training_vocab:test_ppl
Model 2: delta_test_mean ~ training_vocab:test_ppl + sg_score
  Res.Df        RSS Df  Sum of Sq      F Pr(>F)
1     23 9.9763e-05                            
2     22 9.8090e-05  1 1.6733e-06 0.3753 0.5464

Call:
lm(formula = delta_test_mean ~ training_vocab:test_ppl + sg_score, 
    data = regression_data)

Residuals:
       Min         1Q     Median         3Q        Max 
-0.0024892 -0.0010960 -0.0008547  0.0012586  0.0035102 

Coefficients:
                                  Estimate Std. Error t value Pr(>|t|)  
(Intercept)                      5.512e-03  2.319e-03   2.377   0.0266 *
sg_score                         2.023e-03  3.303e-03   0.613   0.5464  
training_vocabbllip-lg:test_ppl -1.343e-05  2.149e-05  -0.625   0.5382  
training_vocabbllip-md:test_ppl -1.970e-05  1.791e-05  -1.100   0.2831  
training_vocabbllip-sm:test_ppl -2.326e-05  1.595e-05  -1.459   0.1588  
training_vocabbllip-xs:test_ppl -2.030e-05  1.064e-05  -1.908   0.0695 .
training_vocabgptbpe:test_ppl    1.007e-06  5.228e-06   0.193   0.8490  
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 0.002112 on 22 degrees of freedom
Multiple R-squared:  0.3384,    Adjusted R-squared:  0.158 
F-statistic: 1.876 on 6 and 22 DF,  p-value: 0.1305
do_stepwise_regression("natural-stories")
[1] "----------------------"
[1] "natural-stories"
Analysis of Variance Table

Model 1: delta_test_mean ~ training_vocab:test_ppl
Model 2: delta_test_mean ~ training_vocab:test_ppl + sg_score
  Res.Df        RSS Df Sum of Sq      F Pr(>F)
1     23 1.2704e-05                           
2     22 1.2284e-05  1   4.2e-07 0.7522 0.3951

Call:
lm(formula = delta_test_mean ~ training_vocab:test_ppl + sg_score, 
    data = regression_data)

Residuals:
       Min         1Q     Median         3Q        Max 
-1.244e-03 -5.810e-04  6.539e-05  4.671e-04  1.062e-03 

Coefficients:
                                  Estimate Std. Error t value Pr(>|t|)    
(Intercept)                      6.636e-03  8.207e-04   8.086 4.94e-08 ***
sg_score                         1.014e-03  1.169e-03   0.867 0.395139    
training_vocabbllip-lg:test_ppl -3.158e-05  7.604e-06  -4.153 0.000415 ***
training_vocabbllip-md:test_ppl -2.918e-05  6.337e-06  -4.604 0.000138 ***
training_vocabbllip-sm:test_ppl -3.327e-05  5.643e-06  -5.895 6.23e-06 ***
training_vocabbllip-xs:test_ppl -2.288e-05  3.765e-06  -6.076 4.08e-06 ***
training_vocabgptbpe:test_ppl   -3.901e-06  1.850e-06  -2.109 0.046589 *  
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 0.0007472 on 22 degrees of freedom
Multiple R-squared:  0.802, Adjusted R-squared:  0.748 
F-statistic: 14.85 on 6 and 22 DF,  p-value: 9.573e-07

Predictive power and perplexity

model_deltas %>%
  ggplot(aes(x=test_ppl, y=delta_test_mean, color=training_vocab)) +
    geom_errorbar(aes(ymin=delta_test_mean-delta_test_sem, ymax=delta_test_mean+delta_test_sem), alpha=0.4) +
    #geom_smooth(method="lm", se=F) +
    geom_point(stat="identity", position="dodge", alpha=1, size=4, aes(shape=model)) +
    ylab(metric) +
    xlab("Test Perplexity") +
    #coord_cartesian(ylim = c(1, 16)) +
    ggtitle("Test Perplexity vs. Predictive Power") +
    scale_color_manual(values = c("bllip-lg"="#440154FF",
                                  "bllip-md"="#39568CFF",
                                  "bllip-sm"="#1F968BFF",
                                  "bllip-xs"="#73D055FF",
                                  "gptbpe"="#888888")) +
    facet_grid(~corpus, scales="free") +
    #coord_cartesian(ylim = c(0, 150)) +
    theme(axis.text=element_text(size=12),
          strip.text.x = element_text(size=12),
          legend.text=element_text(size=12),
          axis.title=element_text(size=12),
          legend.position = "right")
ggsave("../images/cuny2020/ppl_loglik.png",height=4.5,width=11)

Effect of training data size

model_deltas %>%
  mutate(train_size = log(train_size)) %>%
  ggplot(aes(x=train_size, y=delta_test_mean, color=model)) +
    geom_errorbar(aes(ymin=delta_test_mean-delta_test_sem, ymax=delta_test_mean+delta_test_sem), width = 0.1) +
    geom_smooth(method="lm", se=T, alpha=0.5) +
    geom_point(stat="identity", position="dodge", alpha=1, size=3) +
    ylab(metric) +
    xlab("Log Million Training Tokens") +
    ggtitle("Training Size vs. Predictive Power") +
    facet_grid(corpus~model, scales="free") +
    #scale_color_manual(values = c("#A42EF1", "#3894C8")) +
    theme(axis.text=element_text(size=14),
          strip.text.x = element_text(size=14),
          legend.text=element_text(size=14),
          axis.title=element_text(size=18),
          legend.position = "bottom")

#ggsave("./cogsci_images/training_loglik.png",height=5,width=6)
model_deltas %>%
  mutate(train_size = log(train_size)) %>%
  ggplot(aes(x=train_size, y=sg_score, color=model)) +
    geom_smooth(method="lm", se=T, alpha=0.5) +
    geom_point(stat="identity", position="dodge", alpha=1, size=3) +
    ylab("SG SCore") +
    xlab("Log Million Training Tokens") +
    ggtitle("Training Size vs. Syntactic Generalization") +
    #scale_color_manual(values = c("#A42EF1", "#3894C8")) +
    facet_grid(~model, scales="free") +
    theme(axis.text=element_text(size=14),
          strip.text.x = element_text(size=14),
          legend.text=element_text(size=14),
          axis.title=element_text(size=18),
          legend.position = "bottom")

#ggsave("./cogsci_images/training_sg.png",height=5,width=6)

Smith & Levy reproduction

  
all_data %>%
  #filter(surprisal < 15, surprisal > 0) %>%
  ggplot(aes(x=surprisal, y=psychometric, color=training)) +
    stat_smooth(se=T, alpha=0.5) +
    #geom_errorbar(color="black", width=.2, position=position_dodge(width=.9), alpha=0.3) +
    #geom_point(stat="identity", position="dodge", alpha=1, size=3) +
    ylab("Processing Time (ms)") +
    xlab("Surprisal (bits)") +
    ggtitle("Surprisal vs. Reading Time / Gaze Duration") +
    facet_grid(corpus~model, scales = "free") +
    # scale_color_manual(values = c("bllip-lg"="#440154FF",
    #                           "bllip-md"="#39568CFF",
    #                           "bllip-sm"="#1F968BFF",
    #                           "bllip-xs"="#73D055FF",
    #                           "bllip-lg-gptbpe"="#888888",
    #                           "bllip-md-gptbpe"="#888888",
    #                           "bllip-sm-gptbpe"="#888888",
    #                           "bllip-xs-gptbpe"="#888888")) +
    theme(axis.text=element_text(size=14),
          axis.text.y = element_text(size = 10),
          strip.text.x = element_text(size=14),
          legend.text=element_text(size=14),
          axis.title=element_text(size=18),
          legend.position = "right")
ggsave("../images/cuny2020/surp_corr.png",height=4.5,width=12)

full_residuals %>% #filter(model == "gpt2", str_detect(training, "gptbpe")) %>% 
  mutate(training_vocab=as.factor(ifelse(str_detect(training, "gptbpe"), "gptbpe", as.character(training)))) %>% 
  ggplot(aes(x=surprisal, y=resid, color=training)) +
    facet_grid(~corpus) +
    geom_smooth()

full_residuals %>% right_join(baseline_residuals, by=c("corpus", "code", "model", "training", "seed"), suffix=c(".full", ".baseline")) %>% 
  mutate(resid_delta=resid.full - resid.baseline) %>% 
  ggplot(aes(x=surprisal.full, y=resid_delta, color=training)) +
    facet_grid(~corpus) +
    geom_smooth()

LS0tCnRpdGxlOiAiQ1VOWSAyMDIwIEFuYWx5c2lzIgpvdXRwdXQ6IGh0bWxfbm90ZWJvb2sKLS0tCgojIFBhY2thZ2VzIGFuZCB1dGlsaXRpZXMKCmBgYHtyfQpsaWJyYXJ5KHRpZHl2ZXJzZSkKbGlicmFyeShsbWU0KQpsaWJyYXJ5KGxtZXJUZXN0KQpsaWJyYXJ5KGxvZ2dpbmcpCmxpYnJhcnkobXZ0bm9ybSkKbGlicmFyeShtZ2N2KQpgYGAKCmBgYHtyfQojIENvbXB1dGUgdGhlIGxvZy1saWtlbGlob29kIG9mIGEgbmV3IGRhdGFzZXQgdXNpbmcgYSBmaXQgbG1lNCBtb2RlbC4KbG9nTGlrX3Rlc3QgPC0gZnVuY3Rpb24obG0sIHRlc3RfWCwgdGVzdF95KSB7CiAgcHJlZGljdGlvbnMgPC0gcHJlZGljdChsbSwgdGVzdF9YLCByZS5mb3JtPU5BKQogICMgR2V0IHN0ZC5kZXYuIG9mIHJlc2lkdWFsLCBlc3RpbWF0ZWQgZnJvbSB0cmFpbiBkYXRhCiAgc3RkZXYgPC0gc2lnbWEobG0pCiAgIyBGb3IgZWFjaCBwcmVkaWN0aW9uLS1vYnNlcnZhdGlvbiwgZ2V0IHRoZSBkZW5zaXR5IHAob2JzIHwgTihwcmVkaWN0ZWQsIG1vZGVsX3NpZ21hKSkgYW5kIHJlZHVjZQogIGRlbnNpdHkgPC0gc3VtKGRub3JtKHRlc3RfeSwgcHJlZGljdGlvbnMsIHN0ZGV2LCBsb2c9VFJVRSkpCiAgcmV0dXJuKGRlbnNpdHkpCn0KIyBHZXQgcGVyLXByZWRpY3Rpb24gbG9nLWxpa2VsaWhvb2QKbG9nTGlrX3Rlc3RfcGVyIDwtIGZ1bmN0aW9uKGxtLCB0ZXN0X1gsIHRlc3RfeSkgewogIHByZWRpY3Rpb25zIDwtIHByZWRpY3QobG0sIHRlc3RfWCwgcmUuZm9ybT1OQSkKICAjIEdldCBzdGQuZGV2LiBvZiByZXNpZHVhbCwgZXN0aW1hdGVkIGZyb20gdHJhaW4gZGF0YQogIHN0ZGV2IDwtIHNpZ21hKGxtKQogICMgRm9yIGVhY2ggcHJlZGljdGlvbi0tb2JzZXJ2YXRpb24sIGdldCB0aGUgZGVuc2l0eSBwKG9icyB8IE4ocHJlZGljdGVkLCBtb2RlbF9zaWdtYSkpCiAgZGVuc2l0aWVzIDwtIGRub3JtKHRlc3RfeSwgcHJlZGljdGlvbnMsIHN0ZGV2LCBsb2c9VFJVRSkKICByZXR1cm4oZGVuc2l0aWVzKQp9CiMgQ29tcHV0ZSBNU0Ugb2YgYSBuZXcgZGF0YXNldCB1c2luZyBhIGZpdCBsbWU0IG1vZGVsLgptc2VfdGVzdCA8LSBmdW5jdGlvbihsbSwgdGVzdF9YLCB0ZXN0X3kpIHsKICByZXR1cm4obWVhbigocHJlZGljdChsbSwgdGVzdF9YLCByZS5mb3JtPU5BKSAtIHRlc3RfeSkgXiAyKSkKfQojU2FuaXR5IGNoZWNrcwojbXlsbSA8LSBnYW0ocHN5Y2hvbWV0cmljIH4gIHMoc3VycHJpc2FsLCBicyA9ICJjciIsIGsgPSAyMCkgKyBzKHByZXZfc3VycCwgYnMgPSAiY3IiLCBrID0gMjApICsgdGUoZnJlcSwgbGVuLCBicyA9ICJjciIpICsgdGUocHJldl9mcmVxLCBwcmV2X2xlbiwgYnMgPSAiY3IiKSwgZGF0YT10cmFpbl9kYXRhKQojYyhsb2dMaWsobXlsbSksIGxvZ0xpa190ZXN0KG15bG0sIHRyYWluX2RhdGEsIHRyYWluX2RhdGEkcHN5Y2hvbWV0cmljKSkKI2xvZ0xpa190ZXN0KG15bG0sIHRlc3RfZGF0YSwgdGVzdF9kYXRhJHBzeWNob21ldHJpYykKYGBgCgojIERhdGEgbG9hZGluZyBhbmQgcHJlcHJvY2Vzc2luZwoKYGBge3IgTG9hZCBhbmQgcHJlcHJvY2VzcyBkYXRhfQpkYXRhID0gcmVhZC5jc3YoIi4uL2RhdGEvaGFybW9uaXplZF9yZXN1bHRzLmNzdiIpCgphbGxfZGF0YSA9IGRhdGEgJT4lCiAgbXV0YXRlKHNlZWQgPSBhcy5mYWN0b3Ioc2VlZCkpICU+JQogIGdyb3VwX2J5KGNvcnB1cywgbW9kZWwsIHRyYWluaW5nLCBzZWVkKSAlPiUKICAgIG11dGF0ZShwcmV2X3N1cnAgPSBsYWcoc3VycHJpc2FsKSwKICAgICAgICAgcHJldl9jb2RlID0gbGFnKGNvZGUpLAogICAgICAgICBwcmV2X2xlbiA9IGxhZyhsZW4pLAogICAgICAgICBwcmV2X2ZyZXEgPSBsYWcoZnJlcSksCiAgICAgICAgIHByZXZfc3VycCA9IGxhZyhzdXJwcmlzYWwpLAogICAgICAgICAKICAgICAgICAgcHJldjJfZnJlcSA9IGxhZyhwcmV2X2ZyZXEpLAogICAgICAgICBwcmV2Ml9jb2RlID0gbGFnKHByZXZfY29kZSksCiAgICAgICAgIHByZXYyX2xlbiA9IGxhZyhwcmV2X2xlbiksCiAgICAgICAgIHByZXYyX3N1cnAgPSBsYWcocHJldl9zdXJwKSwKICAgICAgICAgCiAgICAgICAgIHByZXYzX2ZyZXEgPSBsYWcocHJldjJfZnJlcSksCiAgICAgICAgIHByZXYzX2NvZGUgPSBsYWcocHJldjJfY29kZSksCiAgICAgICAgIHByZXYzX2xlbiA9IGxhZyhwcmV2Ml9sZW4pLAogICAgICAgICBwcmV2M19zdXJwID0gbGFnKHByZXYyX3N1cnApKSAlPiUKICB1bmdyb3VwKCkgJT4lCiAgCiAgIyBGaWx0ZXIgYmFjayB0aHJlZSBmb3IgdGhlIGR1bmRlZSBjb3JwdXMuIEZpbHRlciBiYWNrIDEgZm9yIGFsbCBvdGhlciBjb3Jwb3JhCiAgIyBOQiB0aGlzIGVmZmVjdGl2ZWx5IHJlbW92ZXMgYWxsIHplcm8tc3VycHJpc2FsIHJvd3MsIHNpbmNlIGVhcmx5LXNlbnRlbmNlIHRva2VucyBkb24ndCBoYXZlIGNvbnRpZ3VvdXMgdG9rZW4gaGlzdG9yeQogIGZpbHRlcigoY29ycHVzID09ICJkdW5kZWUiICYgY29kZSA9PSBwcmV2Ml9jb2RlICsgMikgfCAoY29ycHVzICE9ICJkdW5kZWUiICYgY29kZSA9PSBwcmV2X2NvZGUgKyAxKSkgJT4lCiAgCiAgc2VsZWN0KC1wcmV2X2NvZGUsIC1wcmV2Ml9jb2RlLCAtcHJldjNfY29kZSkgJT4lCiAgZHJvcF9uYSgpCgphbGxfZGF0YSA9IGFsbF9kYXRhICU+JQogIG11dGF0ZSgKICAgIG1vZGVsID0gYXMuY2hhcmFjdGVyKG1vZGVsKSwKICAgIG1vZGVsID0gaWZfZWxzZShtb2RlbCA9PSAiZ3B0LTIiLCAiZ3B0MiIsIG1vZGVsKSwKICAgIG1vZGVsID0gYXMuZmFjdG9yKG1vZGVsKSkKYGBgCgpgYGB7cn0KbWlzc2luZ19yb3dzID0gYWxsX2RhdGEgJT4lIGNvbXBsZXRlKG5lc3RpbmcoY29ycHVzLCBjb2RlKSwgbmVzdGluZyhtb2RlbCwgdHJhaW5pbmcsIHNlZWQpKSAlPiUgCiAgZ3JvdXBfYnkoY29ycHVzLCBjb2RlKSAlPiUgCiAgICBmaWx0ZXIoc3VtKGlzLm5hKHN1cnByaXNhbCkpID4gMCkgJT4lIAogIHVuZ3JvdXAoKSAlPiUgCiAgYW50aV9qb2luKGFsbF9kYXRhLCBieT1jKCJjb3JwdXMiLCAiY29kZSIsICJtb2RlbCIsICJ0cmFpbmluZyIsICJzZWVkIikpCgptaXNzaW5nX3Jvd3MgJT4lIGdncGxvdChhZXMoeD1jb3JwdXMsIGZpbGw9ZmFjdG9yKHBhc3RlKG1vZGVsLHRyYWluaW5nKSkpKSArIGdlb21fYmFyKHBvc2l0aW9uPXBvc2l0aW9uX2RvZGdlKHdpZHRoPTAuOCkpCnByaW50KG1pc3Npbmdfcm93cyAlPiUgZ3JvdXBfYnkobW9kZWwsIHRyYWluaW5nLCBzZWVkLCBjb3JwdXMpICU+JSBzdW1tYXJpc2Uobj1uKCkpKSAlPiUgYXJyYW5nZShkZXNjKG4pKQpgYGAKCgpgYGB7ciBEcm9wIHRva2VucyBmb3Igd2hpY2ggYW55IG1vZGVsIGlzIG1pc3Npbmcgc3VycHJpc2FsIGRhdGEufQoKIyBDb21wdXRlIHRoZSBpZGVhbCBudW1iZXIgb2YgbW9kZWwtLXNlZWQtLXRyYWluaW5nIG9ic2VydmF0aW9ucyBwZXIgdG9rZW4uCmlkZWFsX3Rva2VuX29ic19jb3VudCA9IGFsbF9kYXRhICU+JSBncm91cF9ieShjb3JwdXMsIGNvZGUpICU+JSBncm91cF9zaXplKCkgJT4lIG1heCgpCgp0b19kcm9wID0gYWxsX2RhdGEgJT4lIGdyb3VwX2J5KGNvcnB1cywgY29kZSkgJT4lIGZpbHRlcihuKCkgIT0gaWRlYWxfdG9rZW5fb2JzX2NvdW50KSAlPiUgdW5ncm91cCgpCmxvZ2luZm8ocGFzdGUoIkRyb3BwaW5nIiwgbnJvdyh0b19kcm9wKSwgIm9ic2VydmF0aW9ucyBjb3JyZXNwb25kaW5nIHRvIGNvcnB1cyB0b2tlbnMgd2hpY2ggYXJlIG1pc3Npbmcgb2JzZXJ2YXRpb25zIGZvciBzb21lIG1vZGVsLiIpKQpsb2dpbmZvKHBhc3RlKCJEcm9wcGluZyIsIHRvX2Ryb3AgJT4lIGdyb3VwX2J5KGNvcnB1cywgY29kZSkgJT4lIG5fZ3JvdXBzKCksICJ0b2tlbnMgd2hpY2ggYXJlIG1pc3Npbmcgb2JzZXJ2YXRpb25zIGZvciBzb21lIG1vZGVsLiIpKQoKYWxsX2RhdGEgPSBhbGxfZGF0YSAlPiUgYW50aV9qb2luKHRvX2Ryb3AgJT4lIGdyb3VwX2J5KGNvcnB1cywgY29kZSksIGJ5PWMoImNvcnB1cyIsICJjb2RlIikpCmxvZ2luZm8ocGFzdGUoIkFmdGVyIGRyb3AsIiwgbnJvdyhhbGxfZGF0YSksICJvYnNlcnZhdGlvbnMgKCIsIGFsbF9kYXRhICU+JSBncm91cF9ieShjb3JwdXMsIGNvZGUpICU+JSBuX2dyb3VwcygpLCAiIHRva2VucykgcmVtYWluLiIpKQpgYGAKCmBgYHtyIERyb3AgdG9rZW5zIGZvciB3aGljaCBhbnkgbW9kZWwgaGFzIHplcm8tdmFsdWVkIHN1cnByaXNhbHMufQoKdG9fZHJvcF96ZXJvcyA9IGFsbF9kYXRhICU+JSBncm91cF9ieShjb3JwdXMsIGNvZGUpICU+JSBmaWx0ZXIoYW55KHN1cnByaXNhbCA9PSAwKSkgJT4lIHVuZ3JvdXAoKQpsb2dpbmZvKHBhc3RlKCJEcm9wcGluZyIsIG5yb3codG9fZHJvcF96ZXJvcyksICJvYnNlcnZhdGlvbnMgY29ycmVzcG9uZGluZyB0byBjb3JwdXMgdG9rZW5zIHdoaWNoIGhhdmUgc3VycHJpc2FsIHplcm9zIGZvciBzb21lIG1vZGVsLiIpKQpsb2dpbmZvKHBhc3RlKCJEcm9wcGluZyIsIHRvX2Ryb3BfemVyb3MgJT4lIGdyb3VwX2J5KGNvcnB1cywgY29kZSkgJT4lIG5fZ3JvdXBzKCksICJ0b2tlbnMgd2hpY2ggaGF2ZSBzdXJwcmlzYWwgemVyb3MgZm9yIHNvbWUgbW9kZWwuIikpCgphbGxfZGF0YSA9IGFsbF9kYXRhICU+JSBhbnRpX2pvaW4odG9fZHJvcF96ZXJvcyAlPiUgZ3JvdXBfYnkoY29ycHVzLCBjb2RlKSwgYnk9YygiY29ycHVzIiwgImNvZGUiKSkKbG9naW5mbyhwYXN0ZSgiQWZ0ZXIgZHJvcCwiLCBucm93KGFsbF9kYXRhKSwgIm9ic2VydmF0aW9ucyAoIiwgYWxsX2RhdGEgJT4lIGdyb3VwX2J5KGNvcnB1cywgY29kZSkgJT4lIG5fZ3JvdXBzKCksICIgdG9rZW5zKSByZW1haW4uIikpCmBgYAoKIAojIExlYXJuIG1vZGVscwogCmBgYHtyfQojIENvbXB1dGUgbGluZWFyIG1vZGVsIHN0YXRzIGZvciB0aGUgZ2l2ZW4gdHJhaW5pbmcgZGF0YSBzdWJzZXQgYW5kIGZ1bGwgdGVzdCBkYXRhLgojIEF1dG9tYXRpY2FsbHkgc3Vic2V0cyB0aGUgdGVzdCBkYXRhIHRvIG1hdGNoIHRoZSByZWxldmFudCBncm91cCBmb3Igd2hpY2ggd2UgYXJlIHRyYWluaW5nIGEgbGluZWFyIG1vZGVsLgpnZXRfbG1fZGF0YSA8LSBmdW5jdGlvbihkZiwgdGVzdF9kYXRhLCBmb3JtdWxhLCBmb2xkLCBzdG9yZV9lbnYpIHsKICAjdGhpc19sbSA8LSBnYW0oZm9ybXVsYSwgZGF0YT1kZik7CiAgdGhpc19sbSA9IGxtKGZvcm11bGEsIGRhdGE9ZGYpCiAgdGhpc190ZXN0X2RhdGEgPC0gc2VtaV9qb2luKHRlc3RfZGF0YSwgZGYsIGJ5PWMoInRyYWluaW5nIiwgIm1vZGVsIiwgInNlZWQiLCAiY29ycHVzIikpOwogIAogICMgU2F2ZSBsbSB0byB0aGUgZ2xvYmFsIGVudiBzbyB0aGF0IHdlIGNhbiBhY2Nlc3MgcmVzaWR1YWxzIGxhdGVyLgogIGxtX25hbWUgPSBwYXN0ZSh1bmlxdWUocGFzdGUoZGYkbW9kZWwsIGRmJHRyYWluaW5nLCBkZiRzZWVkLCBkZiRjb3JwdXMpKVsxXSwgZm9sZCkKICBhc3NpZ24obG1fbmFtZSwgdGhpc19sbSwgZW52aXI9c3RvcmVfZW52KQogIAogIHN1bW1hcmlzZShkZiwKICAgICAgICAgICAgbG9nX2xpayA9IGFzLm51bWVyaWMobG9nTGlrKHRoaXNfbG0sIFJFTUwgPSBGKSksCiAgICAgICAgICAgIHRlc3RfbGlrID0gbG9nTGlrX3Rlc3QodGhpc19sbSwgdGhpc190ZXN0X2RhdGEsIHRoaXNfdGVzdF9kYXRhJHBzeWNob21ldHJpYyksCiAgICAgICAgICAgIHRlc3RfbXNlID0gbXNlX3Rlc3QodGhpc19sbSwgdGhpc190ZXN0X2RhdGEsIHRoaXNfdGVzdF9kYXRhJHBzeWNob21ldHJpYykpCn0KIyBGb3IgYSBwcmV2aW91c2x5IGZpdHRlZCBsbSBzdG9yZWQgaW4gc3RvcmVfZW52LCBnZXQgdGhlIHJlc2lkdWFscyBvbiB0ZXN0IGRhdGEgb2YgdGhlIHJlbGV2YW50IGRhdGEgc3Vic2V0LgpnZXRfbG1fcmVzaWR1YWxzIDwtIGZ1bmN0aW9uKGRmLCBmb2xkLCBzdG9yZV9lbnYpIHsKICAjIFJldHJpZXZlIHRoZSByZWxldmFudCBsbS4KICBsbV9uYW1lID0gcGFzdGUodW5pcXVlKHBhc3RlKGRmJG1vZGVsLCBkZiR0cmFpbmluZywgZGYkc2VlZCwgZGYkY29ycHVzKSlbMV0sIGZvbGQpCiAgdGhpc19sbSA8LSBnZXQobG1fbmFtZSwgZW52aXI9c3RvcmVfZW52KQogIAogIG11dGF0ZShkZiwKICAgICAgICAgbGlrZWxpaG9vZCA9IGxvZ0xpa190ZXN0X3Blcih0aGlzX2xtLCBkZiwgZGYkcHN5Y2hvbWV0cmljKSwKICAgICAgICAgcmVzaWQgPSBkZiRwc3ljaG9tZXRyaWMgLSBwcmVkaWN0KHRoaXNfbG0sIGRmLCByZS5mb3JtPU5BKSkKfQojIENvbXB1dGUgcGVyLWV4YW1wbGUgZGVsdGEtbG9nLWxpa2VsaWhvb2QgZm9yIHRoZSBnaXZlbiB0ZXN0IGZvbGQuCmdldF9sbV9kZWx0YV9sb2dfbGlrIDwtIGZ1bmN0aW9uKHRlc3RfZGF0YSwgZm9sZCwgYmFzZWxpbmVfZW52LCBmdWxsX2VudikgewogIGxtX25hbWUgPSBwYXN0ZSh1bmlxdWUocGFzdGUodGVzdF9kYXRhJG1vZGVsLCB0ZXN0X2RhdGEkdHJhaW5pbmcsIHRlc3RfZGF0YSRzZWVkLCB0ZXN0X2RhdGEkY29ycHVzKSlbMV0sIGZvbGQpCiAgYmFzZWxpbmVfbG0gPC0gZ2V0KGxtX25hbWUsIGVudmlyPWJhc2VsaW5lX2VudikKICBmdWxsX2xtIDwtIGdldChsbV9uYW1lLCBlbnZpcj1mdWxsX2VudikKICAKICBkZWx0YV9sb2dfbGlrID0gbG9nTGlrX3Rlc3RfcGVyKGZ1bGxfbG0sIHRlc3RfZGF0YSwgdGVzdF9kYXRhJHBzeWNob21ldHJpYykgLSBsb2dMaWtfdGVzdF9wZXIoYmFzZWxpbmVfbG0sIHRlc3RfZGF0YSwgdGVzdF9kYXRhJHBzeWNob21ldHJpYykKICByZXR1cm4oY2JpbmQodGVzdF9kYXRhLCBkZWx0YV9sb2dfbGlrPWRlbHRhX2xvZ19saWspKQp9CiMjIyMjCiMgRGVmaW5lIHJlZ3Jlc3Npb24gZm9ybXVsYWUuCiMgRXllLXRyYWNraW5nIHJlZ3Jlc3Npb246IG9ubHkgdXNlIHN1cnByaXNhbCBhbmQgcHJldmlvdXMgc3VycHJpc2FsOyBTUFJUIHJlZ3Jlc3Npb246IHVzZSAyLWJhY2sgZmVhdHVyZXMuCiNiYXNlbGluZV9ydF9yZWdyZXNzaW9uID0gcHN5Y2hvbWV0cmljIH4gdGUoZnJlcSwgbGVuLCBicyA9ICJjciIpICsgdGUocHJldl9mcmVxLCBwcmV2X2xlbiwgYnMgPSAiY3IiKQojYmFzZWxpZV9zcHJ0X3JlZ3Jlc3Npb24gPSBwc3ljaG9tZXRyaWMgfiB0ZShmcmVxLCBsZW4sIGJzID0gImNyIikgKyB0ZShwcmV2X2ZyZXEsIHByZXZfbGVuLCBicyA9ICJjciIpICsgdGUocHJldjJfZnJlcSwgcHJldjJfbGVuLCBicyA9ICJjciIpCiNmdWxsX3J0X3JlZ3Jlc3Npb24gPSAocHN5Y2hvbWV0cmljIH4gcyhzdXJwcmlzYWwsIGJzID0gImNyIiwgayA9IDIwKSArIHMocHJldl9zdXJwLCBicyA9ICJjciIsIGsgPSAyMCkKICAgICAgICAgICAgICAgICAgICAgIysgdGUoZnJlcSwgbGVuLCBicyA9ICJjciIpICsgdGUocHJldl9mcmVxLCBwcmV2X2xlbiwgYnMgPSAiY3IiKSkKI2Z1bGxfc3BydF9yZWdyZXNzaW9uID0gKHBzeWNob21ldHJpYyB+IHMoc3VycHJpc2FsLCBicyA9ICJjciIsIGsgPSAyMCkgKyBzKHByZXZfc3VycCwgYnMgPSAiY3IiLCBrID0gMjApICsgcyhwcmV2Ml9zdXJwLCBicyA9ICJjciIsIGsgPSAyMCkKICAgICAgICAgICAgICAgICAgICAgICAgIysgdGUoZnJlcSwgbGVuLCBicyA9ICJjciIpICsgdGUocHJldl9mcmVxLCBwcmV2X2xlbiwgYnMgPSAiY3IiKSArIHRlKHByZXYyX2ZyZXEsIHByZXYyX2xlbiwgYnMgPSAiY3IiKSkKCmJhc2VsaW5lX3J0X3JlZ3Jlc3Npb24gPSBwc3ljaG9tZXRyaWMgfiBmcmVxICsgcHJldl9mcmVxICsgcHJldjJfZnJlcSArIHByZXYzX2ZyZXEgKyBsZW4gKyBwcmV2X2xlbiArIHByZXYyX2xlbiArIHByZXYzX2xlbgpiYXNlbGluZV9zcHJ0X3JlZ3Jlc3Npb24gPSBwc3ljaG9tZXRyaWMgfiBmcmVxICsgcHJldl9mcmVxICsgbGVuICsgcHJldl9sZW4KCmZ1bGxfcnRfcmVncmVzc2lvbiA9IHBzeWNob21ldHJpYyB+IHN1cnByaXNhbCArIHByZXZfc3VycCArIHByZXYyX3N1cnAgKyBwcmV2M19zdXJwICsgZnJlcSArIHByZXZfZnJlcSArIHByZXYyX2ZyZXEgKyBwcmV2M19mcmVxICsgbGVuICsgcHJldl9sZW4gKyBwcmV2Ml9sZW4gKyBwcmV2M19sZW4KZnVsbF9zcHJ0X3JlZ3Jlc3Npb24gPSBwc3ljaG9tZXRyaWMgfiBzdXJwcmlzYWwgKyBwcmV2X3N1cnAgKyBmcmVxICsgcHJldl9mcmVxICsgbGVuICsgcHJldl9sZW4KICAKIyMjIyMKIyBQcmVwYXJlIGZyYW1lcy9lbnZpcm9ubWVudHMgZm9yIHN0b3JpbmcgcmVzdWx0cy9vYmplY3RzLgpiYXNlbGluZV9yZXN1bHRzID0gZGF0YS5mcmFtZSgpCmZ1bGxfbW9kZWxfcmVzdWx0cyA9IGRhdGEuZnJhbWUoKQpiYXNlbGluZV9yZXNpZHVhbHMgPSBkYXRhLmZyYW1lKCkKZnVsbF9yZXNpZHVhbHMgPSBkYXRhLmZyYW1lKCkKbG9nX2xpa19kZWx0YXMgPSBkYXRhLmZyYW1lKCkKCiNSYW5kb21seSBzaHVmZmxlIHRoZSBkYXRhCmFsbF9kYXRhPC1hbGxfZGF0YVtzYW1wbGUobnJvdyhhbGxfZGF0YSkpLF0KI0NyZWF0ZSBLIGVxdWFsbHkgc2l6ZSBmb2xkcwpLID0gNQpmb2xkcyA8LSBjdXQoc2VxKDEsbnJvdyhhbGxfZGF0YSkpLGJyZWFrcz1LLGxhYmVscz1GQUxTRSkKI1BlcmZvcm0gMTAgZm9sZCBjcm9zcyB2YWxpZGF0aW9uCgojIEZpdCBtb2RlbHMgZm9yIHNvbWUgZm9sZCBvZiB0aGUgZGF0YS4KYmFzZWxpbmVfY29ycHVzID0gZnVuY3Rpb24oY29ycHVzLCBkZiwgdGVzdF9kYXRhLCBmb2xkLCBlbnYpIHsKICBpZihjb3JwdXMgPT0gImR1bmRlZSIpIHsKICAgIGdldF9sbV9kYXRhKGRmLCB0ZXN0X2RhdGEsIGJhc2VsaW5lX3J0X3JlZ3Jlc3Npb24sIGZvbGQsIGVudikKICB9IGVsc2UgewogICAgZ2V0X2xtX2RhdGEoZGYsIHRlc3RfZGF0YSwgYmFzZWxpbmVfc3BydF9yZWdyZXNzaW9uLCBmb2xkLCBlbnYpCiAgfQp9CmZ1bGxfbW9kZWxfY29ycHVzID0gZnVuY3Rpb24oY29ycHVzLCBkZiwgdGVzdF9kYXRhLCBmb2xkLCBlbnYpIHsKICBpZihjb3JwdXNbMV0gPT0gImR1bmRlZSIpIHsKICAgIGdldF9sbV9kYXRhKGRmLCB0ZXN0X2RhdGEsIGZ1bGxfcnRfcmVncmVzc2lvbiwgZm9sZCwgZW52KQogIH0gZWxzZSB7CiAgICBnZXRfbG1fZGF0YShkZiwgdGVzdF9kYXRhLCBmdWxsX3NwcnRfcmVncmVzc2lvbiwgZm9sZCwgZW52KQogIH0KfQoKIyBQcmVwYXJlIGEgbmV3IEVudmlyb25tZW50IGluIHdoaWNoIHdlIHN0b3JlIGZpdHRlZCBMTXMsIHdoaWNoIHdlJ2xsIHF1ZXJ5IGxhdGVyIGZvciByZXNpZHVhbHMgYW5kIG90aGVyIG1ldHJpY3MuCmJhc2VsaW5lX2VudiA9IG5ldy5lbnYoKQpmdWxsX2VudiA9IG5ldy5lbnYoKQoKZm9yKGkgaW4gMTpLKSB7IAogICNTZWdlbWVudCB5b3VyIGRhdGEgYnkgZm9sZCB1c2luZyB0aGUgd2hpY2goKSBmdW5jdGlvbiAKICB0ZXN0SW5kZXhlcyA8LSB3aGljaChmb2xkcz09aSwgYXJyLmluZD1UUlVFKQogIHRlc3RfZGF0YSA8LSBhbGxfZGF0YVt0ZXN0SW5kZXhlcywgXQogIHRyYWluX2RhdGEgPC0gYWxsX2RhdGFbLXRlc3RJbmRleGVzLCBdCiAgCiAgIyBDb21wdXRlIGEgYmFzZWxpbmUgbGluZWFyIG1vZGVsIGZvciBlYWNoIG1vZGVsLS10cmFpbmluZy0tc2VlZC0tUlQtY29ycHVzIGNvbWJpbmF0aW9uLgogIGJhc2VsaW5lcyA9IHRyYWluX2RhdGEgJT4lCiAgICBncm91cF9ieShtb2RlbCwgdHJhaW5pbmcsIHNlZWQsIGNvcnB1cykgJT4lCiAgICAgIHByaW50KG1vZGVsKSAlPiUKICAgICAgZG8oYmFzZWxpbmVfY29ycHVzKHVuaXF1ZSguJGNvcnB1cyksIC4sIHRlc3RfZGF0YSwgaSwgYmFzZWxpbmVfZW52KSkgJT4lCiAgICB1bmdyb3VwKCkgJT4lCiAgICBtdXRhdGUoc2VlZCA9IGFzLmZhY3RvcihzZWVkKSwKICAgICAgICAgICBmb2xkID0gaSkKICAKICBiYXNlbGluZV9yZXN1bHRzID0gcmJpbmQoYmFzZWxpbmVfcmVzdWx0cywgYmFzZWxpbmVzKQogIAogICMgQ29tcHV0ZSBhIGZ1bGwgbGluZWFyIG1vZGVsIGZvciBlYWNoIG1vZGVsLS10cmFpbmluZy0tc2VlZC1SVC1jb3JwdXMgY29tYmluYXRpb24KICBmdWxsX21vZGVscyA9IHRyYWluX2RhdGEgJT4lCiAgICBncm91cF9ieShtb2RlbCwgdHJhaW5pbmcsIHNlZWQsIGNvcnB1cykgJT4lCiAgICAgIGRvKGZ1bGxfbW9kZWxfY29ycHVzKHVuaXF1ZSguJGNvcnB1cyksIC4sIHRlc3RfZGF0YSwgaSwgZnVsbF9lbnYpKSAlPiUKICAgIHVuZ3JvdXAoKSAlPiUKICAgIG11dGF0ZShzZWVkID0gYXMuZmFjdG9yKHNlZWQpLAogICAgICAgICAgIGZvbGQgPSBpKQogIAogIGZ1bGxfbW9kZWxfcmVzdWx0cyA9IHJiaW5kKGZ1bGxfbW9kZWxfcmVzdWx0cywgZnVsbF9tb2RlbHMpCiAgCiAgIyBDb21wdXRlIGRlbHRhLWxvZy1saWtlbGlob29kcwogIGZvbGRfbG9nX2xpa19kZWx0YXMgPSB0ZXN0X2RhdGEgJT4lCiAgICBncm91cF9ieShtb2RlbCwgdHJhaW5pbmcsIHNlZWQsIGNvcnB1cykgJT4lCiAgICAgIGRvKGdldF9sbV9kZWx0YV9sb2dfbGlrKC4sIGksIGJhc2VsaW5lX2VudiwgZnVsbF9lbnYpKSAlPiUKICAgIHVuZ3JvdXAoKQoKICBsb2dfbGlrX2RlbHRhcyA9IHJiaW5kKGxvZ19saWtfZGVsdGFzLCBmb2xkX2xvZ19saWtfZGVsdGFzKQogIAogIGZvbGRfYmFzZWxpbmVfcmVzaWR1YWxzID0gdGVzdF9kYXRhICU+JQogICAgZ3JvdXBfYnkobW9kZWwsIHRyYWluaW5nLCBzZWVkLCBjb3JwdXMpICU+JQogICAgICBkbyhnZXRfbG1fcmVzaWR1YWxzKC4sIGksIGJhc2VsaW5lX2VudikpICU+JQogICAgdW5ncm91cCgpCgogIGJhc2VsaW5lX3Jlc2lkdWFscyA9IHJiaW5kKGJhc2VsaW5lX3Jlc2lkdWFscywgZm9sZF9iYXNlbGluZV9yZXNpZHVhbHMpCgogIGZvbGRfZnVsbF9yZXNpZHVhbHMgPSB0ZXN0X2RhdGEgJT4lCiAgICBncm91cF9ieShtb2RlbCwgdHJhaW5pbmcsIHNlZWQsIGNvcnB1cykgJT4lCiAgICAgIGRvKGdldF9sbV9yZXNpZHVhbHMoLiwgaSwgZnVsbF9lbnYpKSAlPiUKICAgIHVuZ3JvdXAoKQoKICBmdWxsX3Jlc2lkdWFscyA9IHJiaW5kKGZ1bGxfcmVzaWR1YWxzLCBmb2xkX2Z1bGxfcmVzaWR1YWxzKQp9CmBgYAoKYGBge3J9CiN3cml0ZS5jc3YoZnVsbF9yZXNpZHVhbHMsICIuLi9kYXRhL2FuYWx5c2lzX2NoZWNrcG9pbnRzL2Z1bGxfcmVzaWR1YWxzLmNzdiIpCiN3cml0ZS5jc3YoYmFzZWxpbmVfcmVzaWR1YWxzLCAiLi4vZGF0YS9hbmFseXNpc19jaGVja3BvaW50cy9iYXNlbGluZV9yZXNpZHVhbHMuY3N2IikKYGBgCgpgYGB7cn0KbW9kZWxfZGVsdGFzID0gbG9nX2xpa19kZWx0YXMgJT4lCiAgZ3JvdXBfYnkobW9kZWwsIHRyYWluaW5nLCBzZWVkLCBjb3JwdXMpICU+JSAKICBzdW1tYXJpc2UobWVhbl9kZWx0YV9sb2dfbGlrID0gbWVhbihkZWx0YV9sb2dfbGlrKSwKICAgICAgICAgICAgc2VtX2RlbHRhX2xvZ19saWsgPSBzZChkZWx0YV9sb2dfbGlrKSAvIHNxcnQobGVuZ3RoKGRlbHRhX2xvZ19saWspKSkKYGBgCgpgYGB7cn0Kd3JpdGUuY3N2KGZ1bGxfbW9kZWxfcmVzdWx0cywgIi4uL2RhdGEvYW5hbHlzaXNfY2hlY2twb2ludHMvZnVsbF9tb2RlbF9yZXN1bHQuY3N2IikKd3JpdGUuY3N2KGJhc2VsaW5lX3Jlc3VsdHMsICIuLi9kYXRhL2FuYWx5c2lzX2NoZWNrcG9pbnRzL2Jhc2VsaW5lX3Jlc3VsdHMuY3N2IikKI2Z1bGxfbW9kZWxfcmVzdWx0cyA9IHJlYWQuY3N2KCIuLi9kYXRhL2FuYWx5c2lzX2NoZWNrcG9pbnRzL2ZmdWxsX21vZGVsX3Jlc3VsdHMuY3N2IikKI2Jhc2VsaW5lX3Jlc3VsdHMgPSByZWFkLmNzdigiLi4vZGF0YS9hbmFseXNpc19jaGVja3BvaW50cy9mYmFzZWxpbmVfcmVzdWx0c2IuY3N2IikKYGBgCgpgYGB7cn0KbWV0cmljIDwtICLOlExvZ0xpayIKI21ldHJpYyA8LSAiLc6UTVNFIgoKIyAjIFNlbGVjdCB0aGUgcmVsZXZhbnQgbWV0cmljLgptb2RlbF9kZWx0YXMgPSBtb2RlbF9kZWx0YXMgJT4lCiAgICAjIFJldHJpZXZlIHRoZSBjdXJyZW50IHRlc3QgbWV0cmljCiAgICBtdXRhdGUoZGVsdGFfdGVzdF9tZWFuID0gbWVhbl9kZWx0YV9sb2dfbGlrLAogICAgICAgICAgIGRlbHRhX3Rlc3Rfc2VtID0gc2VtX2RlbHRhX2xvZ19saWspICU+JQogICAgIyBtdXRhdGUoZGVsdGFfdGVzdF9tZWFuID0gbWVhbl9kZWx0YV9tc2UsCiAgICAjICAgICAgICBkZWx0YV90ZXN0X3NlbSA9IHNlbV9kZWx0YV9tc2UpCiAgICAKICAgICMgUmVtb3ZlIHRoZSByYXcgbWV0cmljcy4KICAgIHNlbGVjdCgtbWVhbl9kZWx0YV9sb2dfbGlrLCAtc2VtX2RlbHRhX2xvZ19saWssCiAgICAgICAgICAgIy1tZWFuX2RlbHRhX21zZSwgLXNlbV9kZWx0YV9tc2UKICAgICAgICAgICApCm1vZGVsX2RlbHRhcwpgYGAKCmBgYHtyfQojIFNhbml0eSBjaGVjazogdHJhaW5pbmcgb24gdHJhaW4rdGVzdCBkYXRhIHNob3VsZCB5aWVsZCBpbXByb3ZlZCBwZXJmb3JtYW5jZSBvdmVyIHRyYWluaW5nIG9uIGp1c3QgdHJhaW5pbmcgZGF0YS4gKFdoZW4gZXZhbHVhdGluZyBvbiB0ZXN0IGRhdGEuKQojIGZ1bGxfYmFzZWxpbmVzID0gYWxsX2RhdGEgJT4lCiMgICBncm91cF9ieShtb2RlbCwgdHJhaW5pbmcsIHNlZWQsIGNvcnB1cykgJT4lCiMgICBzdW1tYXJpc2UoYmFzZWxpbmVfdHJhaW5fYWxsX3Rlc3RfbGlrID0gbG9nTGlrX3Rlc3QobG0ocHN5Y2hvbWV0cmljIH4gbGVuICsgZnJlcSArIHNlbnRfcG9zLCBkYXRhPS4pLCBzZW1pX2pvaW4odGVzdF9kYXRhLCAuLCBieT1jKCJ0cmFpbmluZyIsICJtb2RlbCIsICJzZWVkIiwgImNvcnB1cyIpKSwgc2VtaV9qb2luKHRlc3RfZGF0YSwgLiwgYnk9YygidHJhaW5pbmciLCAibW9kZWwiLCAic2VlZCIsICJjb3JwdXMiKSkkcHN5Y2hvbWV0cmljKSkgJT4lCiMgICB1bmdyb3VwKCkKIyBmdWxsX2Jhc2VsaW5lcwojIAojIGZ1bGxfYmFzZWxpbmVzICU+JQojICAgcmlnaHRfam9pbihiYXNlbGluZXMsIGJ5PWMoInNlZWQiLCAidHJhaW5pbmciLCAibW9kZWwiLCAiY29ycHVzIikpICU+JQojICAgbXV0YXRlKGRlbHRhPWJhc2VsaW5lX3RyYWluX2FsbF90ZXN0X2xpay1iYXNlbGluZV90ZXN0X2xpaykgJT4lCiMgICBzZWxlY3QoLWJhc2VsaW5lX2xpaykgIyAlPiUKIyAgICNzZWxlY3QoLWJhc2VsaW5lX3Rlc3RfbGlrLCAtYmFzZWxpbmVfdHJhaW5fYWxsX3Rlc3RfbGlrLCAtYmFzZWxpbmVfbGlrLCAtYmFzZWxpbmVfdGVzdF9tc2UpCmBgYAoKIyBMb2FkIGxhbmd1YWdlIG1vZGVsIGRhdGEgKFN5bnRheEd5bSwgUFBMKQoKYGBge3J9Cmxhbmd1YWdlX21vZGVsX2RhdGEgPSByZWFkLmNzdigiLi4vZGF0YS9tb2RlbF9tZXRhZGF0YS5jc3YiKSAlPiUKICBtdXRhdGUobW9kZWwgPSBhcy5jaGFyYWN0ZXIobW9kZWwpLAogICAgICAgICBtb2RlbCA9IGlmX2Vsc2UobW9kZWwgPT0gImdwdC0yIiwgImdwdDIiLCBtb2RlbCksCiAgICAgICAgIG1vZGVsID0gYXMuZmFjdG9yKG1vZGVsKSkgJT4lCiAgbXV0YXRlKHRyYWluX3NpemUgPSBjYXNlX3doZW4oc3RyX3N0YXJ0cyh0cmFpbmluZywgImJsbGlwLWxnIikgfiA0MiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBzdHJfc3RhcnRzKHRyYWluaW5nLCAiYmxsaXAtbWQiKSB+IDE1LAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHN0cl9zdGFydHModHJhaW5pbmcsICJibGxpcC1zbSIpIH4gNSwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBzdHJfc3RhcnRzKHRyYWluaW5nLCAiYmxsaXAteHMiKSB+IDEpLAogICAgICAgICAKICAgICAgICAgIyBUcmFpbmluZyB2b2NhYnVsYXJ5IHVzdWFsbHkgY292YXJpZXMgd2l0aCB0aGUgdHJhaW5pbmcgY29ycHVzLgogICAgICAgICAjIEJ1dCBCUEUgbW9kZWxzIHNoYXJlIGEgdm9jYWJ1bGFyeSBhY3Jvc3MgdHJhaW5pbmcgY29ycG9yYS4KICAgICAgICAgdHJhaW5pbmdfdm9jYWI9YXMuZmFjdG9yKGlmZWxzZShzdHJfZGV0ZWN0KHRyYWluaW5nLCAiZ3B0YnBlIiksICJncHRicGUiLCBhcy5jaGFyYWN0ZXIodHJhaW5pbmcpKSkKICAgICAgICAgKSAlPiUKICBtdXRhdGUoc2VlZCA9IGFzLmZhY3RvcihzZWVkKSkgJT4lCiAgc2VsZWN0KC1waWQsIC10ZXN0X2xvc3MpICU+JQogIGRpc3RpbmN0KG1vZGVsLCB0cmFpbmluZywgc2VlZCwgLmtlZXBfYWxsID0gVFJVRSkKdGFibGUobGFuZ3VhZ2VfbW9kZWxfZGF0YSRzZWVkKQp0YWJsZShtb2RlbF9kZWx0YXMkc2VlZCkKYGBgCgpGaXJzdCBqb2luIGRlbHRhLW1ldHJpYyBkYXRhIHdpdGggbW9kZWwgYXV4aWxpYXJ5IGRhdGEuCgpgYGB7cn0KbW9kZWxfZGVsdGFzID0gbW9kZWxfZGVsdGFzICU+JQogIG1lcmdlKGxhbmd1YWdlX21vZGVsX2RhdGEsIGJ5ID0gYygic2VlZCIsICJ0cmFpbmluZyIsICJtb2RlbCIpLCBhbGw9VCkgJT4lCiAgZHJvcF9uYSgpCgptb2RlbF9kZWx0YXMKYGBgCgpBbHNvIGpvaW4gb24gdGhlIG9yaWdpbmFsIGxpbmVhciBtb2RlbCBkYXRhLCByYXRoZXIgdGhhbiBjb2xsYXBzaW5nIHRvIGRlbHRhLW1ldHJpY3MuClRoaXMgd2lsbCBzdXBwb3J0IHJlZ3Jlc3Npb25zIGxhdGVyIG9uIHRoYXQgZG9uJ3QgY29sbGFwc2UgYWNyb3NzIGZvbGRzLgoKCiMgRmluYWwgZGF0YSBwcmVwcm9jZXNzaW5nCgpgYGB7ciBGaWx0ZXIgbW9kZWxzIGFuZC9vciBjb3Jwb3JhfQojIEV4Y2x1ZGUgb3JkZXJlZC1uZXVyb25zIGZyb20gYWxsIGFuYWx5c2VzLgptb2RlbF9kZWx0YXMgPC0gbW9kZWxfZGVsdGFzICU+JQogIGZpbHRlcihtb2RlbCAhPSAib3JkZXJlZC1uZXVyb25zIikKYGBgCgoKIyBWaXN1YWxpemF0aW9ucwoKIyMgVGhlIGJhc2ljcwoKYGBge3IsIGZpZy5jYXA9IkNvcnB1cyBzaXplcyJ9CmFsbF9kYXRhICU+JSBnZ3Bsb3QoYWVzKHg9Y29ycHVzKSkgKyBnZW9tX2JhcigpCnByaW50KGFsbF9kYXRhICU+JSBncm91cF9ieShjb3JwdXMpICU+JSBzdW1tYXJpc2Uobj1uKCkpKQpgYGAKCgpgYGB7ciwgZmlnLmNhcD0iV29yZCBmcmVxdWVuY3kgZGlzdHJpYnV0aW9uIGJ5IGNvcnB1cyJ9CmFsbF9kYXRhICU+JSAKICBnZ3Bsb3QoYWVzKHg9ZnJlcSwgY29sb3I9Y29ycHVzKSkgKyBnZW9tX2RlbnNpdHkoKQpgYGAKCmBgYHtyLCBmaWcuY2FwPSJXb3JkIGxlbmd0aCBkaXN0cmlidXRpb24gYnkgY29ycHVzIn0KYWxsX2RhdGEgJT4lIAogIGdncGxvdChhZXMoeD1sZW4sIGNvbG9yPWNvcnB1cykpICsgZ2VvbV9kZW5zaXR5KCkKYGBgCgpgYGB7ciwgZmlnLmNhcD0iU3VycHJpc2FsIGRpc3RyaWJ1dGlvbiBieSBjb3JwdXMifQphbGxfZGF0YSAlPiUgCiAgZ2dwbG90KGFlcyh4PXN1cnByaXNhbCwgY29sb3I9Y29ycHVzKSkgKyBnZW9tX2RlbnNpdHkoKQpgYGAKCiMjIFByZWRpY3RpdmUgcG93ZXIgYW5kIFNHCgoKYGBge3IgQnkgbW9kZWx9Cm1vZGVsX2RlbHRhcyAlPiUKICBnZ3Bsb3QoYWVzKHg9c2dfc2NvcmUsIHk9ZGVsdGFfdGVzdF9tZWFuKSkgKwogICAgZ2VvbV9lcnJvcmJhcihhZXMoeW1pbj1kZWx0YV90ZXN0X21lYW4tZGVsdGFfdGVzdF9zZW0sIHltYXg9ZGVsdGFfdGVzdF9tZWFuK2RlbHRhX3Rlc3Rfc2VtKSkgKwogICAgZ2VvbV9zbW9vdGgobWV0aG9kPSJsbSIsIHNlPVQpICsKICAgIGdlb21fcG9pbnQoc3RhdD0iaWRlbnRpdHkiLCBwb3NpdGlvbj0iZG9kZ2UiLCBhbHBoYT0xLCBzaXplPTMsIGFlcyhjb2xvcj10cmFpbmluZ192b2NhYiwgc2hhcGU9bW9kZWwpKSArCiAgICB5bGFiKG1ldHJpYykgKwogICAgeGxhYigiU3ludGF4IEdlbmVyYWxpemF0aW9uIFNjb3JlIikgKwogICAgZ2d0aXRsZSgiU3ludGFjdGljIEdlbmVyYWxpemF0aW9uIHZzLiBQcmVkaWN0aXZlIFBvd2VyIikgKwogICAgc2NhbGVfY29sb3JfbWFudWFsKHZhbHVlcyA9IGMoImJsbGlwLWxnIj0iIzQ0MDE1NEZGIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImJsbGlwLW1kIj0iIzM5NTY4Q0ZGIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImJsbGlwLXNtIj0iIzFGOTY4QkZGIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImJsbGlwLXhzIj0iIzczRDA1NUZGIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImdwdGJwZSI9IiM4ODg4ODgiKSkgKwogICAgZmFjZXRfZ3JpZCh+Y29ycHVzLCBzY2FsZXM9ImZyZWUiKSArCiAgICB0aGVtZShheGlzLnRleHQ9ZWxlbWVudF90ZXh0KHNpemU9MTQpLAogICAgICAgICAgc3RyaXAudGV4dC54ID0gZWxlbWVudF90ZXh0KHNpemU9MTQpLAogICAgICAgICAgbGVnZW5kLnRleHQ9ZWxlbWVudF90ZXh0KHNpemU9MTQpLAogICAgICAgICAgYXhpcy50aXRsZT1lbGVtZW50X3RleHQoc2l6ZT0xOCksCiAgICAgICAgICBsZWdlbmQucG9zaXRpb24gPSAiYm90dG9tIikKI2dnc2F2ZSgiLi9jb2dzY2lfaW1hZ2VzL3NnX2xvZ2xpay5wbmciLGhlaWdodD01LHdpZHRoPTYpCmBgYAoKIyMjIFJlZ3Jlc3Npb24gYW5hbHlzZXMKCldlIGNvbnRyb2wgZm9yIGVmZmVjdHMgb2YgcGVycGxleGl0eSBieSByZWxhdGluZyB0aGUgcmVzaWR1YWxzIG9mIGEgYHBlcmZvcm1hbmNlIH4gUFBMYCByZWdyZXNzaW9uIHRvIFNHIHNjb3JlLgoKYGBge3IgUmVzaWR1YWxpemVkIHJlZ3Jlc3Npb259CiMgUHJlcGFyZSBhIHJlc2lkdWFsaXplZCByZWdyZXNzaW9uIGZvciB4MSBvbnRvIHksIGNvbnRyb2xsaW5nIGZvciB0aGUgZWZmZWN0cyBvZiB4Mi4KZF9yZXNpZCA9IG1vZGVsX2RlbHRhcyAlPiUKICBkcm9wX25hKCkgJT4lCiAgCiAgIyBSZXNpZHVhbGl6ZSBkZWx0YSBtZXRyaWMgdy5yLnQgUFBMIGZvciBlYWNoIG1vZGVsLS10cmFpbmluZy0tc2VlZAogIGdyb3VwX2J5KGNvcnB1cykgJT4lCiAgICBtdXRhdGUocmVzaWQuZGVsdGEgPSByZXNpZChsbShkZWx0YV90ZXN0X21lYW4gfiB0cmFpbmluZzp0ZXN0X3BwbCkpKSAlPiUKICB1bmdyb3VwKCkgJT4lCiAgCiAgIyBSZXNpZHVhbGl6ZSBTRyBzY29yZSB3LnIudC4gUFBMIGZvciBlYWNoIHRyYWluaW5nIHZvY2FidWxhcnkKICBncm91cF9ieSh0cmFpbmluZ192b2NhYikgJT4lCiAgICAjIE5CIG5vIG5lZWQgZm9yIHRyYWluaW5nOnBwbCBpbnRlcmFjdGlvbiwgc2luY2Ugd2UncmUgd2l0aGluLWdyb3VwLgogICAgbXV0YXRlKHJlc2lkLnNnID0gcmVzaWQobG0oc2dfc2NvcmUgfiB0ZXN0X3BwbCkpKSAlPiUKICB1bmdyb3VwKCkgJT4lCiAgIyBDb21wdXRlIHN1bW1hcnkgc3RhdGlzdGljcyBhY3Jvc3MgbW9kZWwtLXRyYWluaW5nLS1zZWVkLS1jb3JwdXMuCiAgZ3JvdXBfYnkobW9kZWwsIHRyYWluaW5nX3ZvY2FiLCBjb3JwdXMsIHNlZWQpICU+JQogICAgc3VtbWFyaXNlKHJlc2lkLmRlbHRhLm1lYW4gPSBtZWFuKHJlc2lkLmRlbHRhKSwKICAgICAgICAgICAgICByZXNpZC5kZWx0YS5zZW0gPSBzZChyZXNpZC5kZWx0YSkgLyBzcXJ0KGxlbmd0aChyZXNpZC5kZWx0YSkpLAogICAgICAgICAgICAgIHJlc2lkLnNnLm1lYW4gPSBtZWFuKHJlc2lkLnNnKSwKICAgICAgICAgICAgICByZXNpZC5zZy5zZW0gPSBzZChyZXNpZC5zZykgLyBzcXJ0KGxlbmd0aChyZXNpZC5zZykpKQojIE5vdyBwbG90IHJlc2lkdWFsIHZzIFNHCmRfcmVzaWQgJT4lCiAgI2ZpbHRlcihjb3JwdXMgIT0gImJuYy1icm93biIpICU+JQogIGdncGxvdChhZXMoeD1yZXNpZC5zZy5tZWFuLCB5PXJlc2lkLmRlbHRhLm1lYW4pKSArCiAgICBnZW9tX2Vycm9yYmFyKGFlcyh4bWluPXJlc2lkLnNnLm1lYW4gLSByZXNpZC5zZy5zZW0sCiAgICAgICAgICAgICAgICAgICAgICB4bWF4PXJlc2lkLnNnLm1lYW4gKyByZXNpZC5zZy5zZW0sCiAgICAgICAgICAgICAgICAgICAgICB5bWluPXJlc2lkLmRlbHRhLm1lYW4gLSByZXNpZC5kZWx0YS5zZW0sCiAgICAgICAgICAgICAgICAgICAgICB5bWF4PXJlc2lkLmRlbHRhLm1lYW4gKyByZXNpZC5kZWx0YS5zZW0pLCBhbHBoYT0wLjMpICsKICAgIGdlb21fc21vb3RoKG1ldGhvZD0ibG0iLCBzZT1UKSArCiAgICBnZW9tX3BvaW50KHN0YXQ9ImlkZW50aXR5IiwgcG9zaXRpb249ImRvZGdlIiwgYWxwaGE9MSwgc2l6ZT00LCBhZXMoc2hhcGU9bW9kZWwsIGNvbG9yPXRyYWluaW5nX3ZvY2FiKSkgKwogICAgeWxhYihwYXN0ZSgiUmVzaWR1YWwiLCBtZXRyaWMpKSArCiAgICB4bGFiKCJSZXNpZHVhbCBTeW50YXggR2VuZXJhbGl6YXRpb24gU2NvcmUiKSArCiAgICBnZ3RpdGxlKCJTeW50YWN0aWMgR2VuZXJhbGl6YXRpb24gdnMuIFByZWRpY3RpdmUgUG93ZXIiKSArCiAgICBzY2FsZV9jb2xvcl9tYW51YWwodmFsdWVzID0gYygiYmxsaXAtbGciPSIjNDQwMTU0RkYiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImJsbGlwLW1kIj0iIzM5NTY4Q0ZGIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJibGxpcC1zbSI9IiMxRjk2OEJGRiIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiYmxsaXAteHMiPSIjNzNEMDU1RkYiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImdwdGJwZSI9IiM4ODg4ODgiKSkgKwogICAgZmFjZXRfZ3JpZCgufmNvcnB1cywgc2NhbGVzPSJmcmVlIikgKwogICAgdGhlbWUoYXhpcy50ZXh0PWVsZW1lbnRfdGV4dChzaXplPTE0KSwKICAgICAgICAgIHN0cmlwLnRleHQueCA9IGVsZW1lbnRfdGV4dChzaXplPTE0KSwKICAgICAgICAgIGxlZ2VuZC50ZXh0PWVsZW1lbnRfdGV4dChzaXplPTE0KSwKICAgICAgICAgIGF4aXMudGl0bGU9ZWxlbWVudF90ZXh0KHNpemU9MTgpLAogICAgICAgICAgbGVnZW5kLnBvc2l0aW9uID0gInJpZ2h0IikKZ2dzYXZlKCIuLi9pbWFnZXMvY3VueTIwMjAvcHBsX3NnLnBuZyIsaGVpZ2h0PTQuNSx3aWR0aD0xMSkKYGBgCgoKYGBge3IgU3RlcHdpc2UgcmVncmVzc2lvbn0KZG9fc3RlcHdpc2VfcmVncmVzc2lvbiA9IGZ1bmN0aW9uKGN1cl9jb3JwdXMpIHsKICByZWdyZXNzaW9uX2RhdGEgPSBtb2RlbF9kZWx0YXMgJT4lCiAgICBmaWx0ZXIoY29ycHVzID09IGN1cl9jb3JwdXMpCiAgCiAgcHJpbnQoIi0tLS0tLS0tLS0tLS0tLS0tLS0tLS0iKQogIHByaW50KGN1cl9jb3JwdXMpCiAgCiAgbG0xID0gbG0oZGVsdGFfdGVzdF9tZWFuIH4gdHJhaW5pbmdfdm9jYWI6dGVzdF9wcGwsIGRhdGEgPSByZWdyZXNzaW9uX2RhdGEpCiAgbG0yID0gbG0oZGVsdGFfdGVzdF9tZWFuIH4gdHJhaW5pbmdfdm9jYWI6dGVzdF9wcGwgKyBzZ19zY29yZSwgZGF0YSA9IHJlZ3Jlc3Npb25fZGF0YSkKICBwcmludChhbm92YShsbTEsIGxtMikpCiAgc3VtbWFyeShsbTIpCn0KI2RvX3N0ZXB3aXNlX3JlZ3Jlc3Npb24oImJuYy1icm93biIpCmRvX3N0ZXB3aXNlX3JlZ3Jlc3Npb24oImR1bmRlZSIpCmRvX3N0ZXB3aXNlX3JlZ3Jlc3Npb24oIm5hdHVyYWwtc3RvcmllcyIpCmBgYAoKIyMgUHJlZGljdGl2ZSBwb3dlciBhbmQgcGVycGxleGl0eQoKYGBge3J9Cm1vZGVsX2RlbHRhcyAlPiUKICBnZ3Bsb3QoYWVzKHg9dGVzdF9wcGwsIHk9ZGVsdGFfdGVzdF9tZWFuLCBjb2xvcj10cmFpbmluZ192b2NhYikpICsKICAgIGdlb21fZXJyb3JiYXIoYWVzKHltaW49ZGVsdGFfdGVzdF9tZWFuLWRlbHRhX3Rlc3Rfc2VtLCB5bWF4PWRlbHRhX3Rlc3RfbWVhbitkZWx0YV90ZXN0X3NlbSksIGFscGhhPTAuNCkgKwogICAgI2dlb21fc21vb3RoKG1ldGhvZD0ibG0iLCBzZT1GKSArCiAgICBnZW9tX3BvaW50KHN0YXQ9ImlkZW50aXR5IiwgcG9zaXRpb249ImRvZGdlIiwgYWxwaGE9MSwgc2l6ZT00LCBhZXMoc2hhcGU9bW9kZWwpKSArCiAgICB5bGFiKG1ldHJpYykgKwogICAgeGxhYigiVGVzdCBQZXJwbGV4aXR5IikgKwogICAgI2Nvb3JkX2NhcnRlc2lhbih5bGltID0gYygxLCAxNikpICsKICAgIGdndGl0bGUoIlRlc3QgUGVycGxleGl0eSB2cy4gUHJlZGljdGl2ZSBQb3dlciIpICsKICAgIHNjYWxlX2NvbG9yX21hbnVhbCh2YWx1ZXMgPSBjKCJibGxpcC1sZyI9IiM0NDAxNTRGRiIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiYmxsaXAtbWQiPSIjMzk1NjhDRkYiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImJsbGlwLXNtIj0iIzFGOTY4QkZGIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJibGxpcC14cyI9IiM3M0QwNTVGRiIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiZ3B0YnBlIj0iIzg4ODg4OCIpKSArCiAgICBmYWNldF9ncmlkKH5jb3JwdXMsIHNjYWxlcz0iZnJlZSIpICsKICAgICNjb29yZF9jYXJ0ZXNpYW4oeWxpbSA9IGMoMCwgMTUwKSkgKwogICAgdGhlbWUoYXhpcy50ZXh0PWVsZW1lbnRfdGV4dChzaXplPTEyKSwKICAgICAgICAgIHN0cmlwLnRleHQueCA9IGVsZW1lbnRfdGV4dChzaXplPTEyKSwKICAgICAgICAgIGxlZ2VuZC50ZXh0PWVsZW1lbnRfdGV4dChzaXplPTEyKSwKICAgICAgICAgIGF4aXMudGl0bGU9ZWxlbWVudF90ZXh0KHNpemU9MTIpLAogICAgICAgICAgbGVnZW5kLnBvc2l0aW9uID0gInJpZ2h0IikKZ2dzYXZlKCIuLi9pbWFnZXMvY3VueTIwMjAvcHBsX2xvZ2xpay5wbmciLGhlaWdodD00LjUsd2lkdGg9MTEpCgpgYGAKCgojIyBFZmZlY3Qgb2YgdHJhaW5pbmcgZGF0YSBzaXplCgpgYGB7ciBPbiBwcmVkaWN0aXZlIHBvd2VyfQptb2RlbF9kZWx0YXMgJT4lCiAgbXV0YXRlKHRyYWluX3NpemUgPSBsb2codHJhaW5fc2l6ZSkpICU+JQogIGdncGxvdChhZXMoeD10cmFpbl9zaXplLCB5PWRlbHRhX3Rlc3RfbWVhbiwgY29sb3I9bW9kZWwpKSArCiAgICBnZW9tX2Vycm9yYmFyKGFlcyh5bWluPWRlbHRhX3Rlc3RfbWVhbi1kZWx0YV90ZXN0X3NlbSwgeW1heD1kZWx0YV90ZXN0X21lYW4rZGVsdGFfdGVzdF9zZW0pLCB3aWR0aCA9IDAuMSkgKwogICAgZ2VvbV9zbW9vdGgobWV0aG9kPSJsbSIsIHNlPVQsIGFscGhhPTAuNSkgKwogICAgZ2VvbV9wb2ludChzdGF0PSJpZGVudGl0eSIsIHBvc2l0aW9uPSJkb2RnZSIsIGFscGhhPTEsIHNpemU9MykgKwogICAgeWxhYihtZXRyaWMpICsKICAgIHhsYWIoIkxvZyBNaWxsaW9uIFRyYWluaW5nIFRva2VucyIpICsKICAgIGdndGl0bGUoIlRyYWluaW5nIFNpemUgdnMuIFByZWRpY3RpdmUgUG93ZXIiKSArCiAgICBmYWNldF9ncmlkKGNvcnB1c35tb2RlbCwgc2NhbGVzPSJmcmVlIikgKwogICAgI3NjYWxlX2NvbG9yX21hbnVhbCh2YWx1ZXMgPSBjKCIjQTQyRUYxIiwgIiMzODk0QzgiKSkgKwogICAgdGhlbWUoYXhpcy50ZXh0PWVsZW1lbnRfdGV4dChzaXplPTE0KSwKICAgICAgICAgIHN0cmlwLnRleHQueCA9IGVsZW1lbnRfdGV4dChzaXplPTE0KSwKICAgICAgICAgIGxlZ2VuZC50ZXh0PWVsZW1lbnRfdGV4dChzaXplPTE0KSwKICAgICAgICAgIGF4aXMudGl0bGU9ZWxlbWVudF90ZXh0KHNpemU9MTgpLAogICAgICAgICAgbGVnZW5kLnBvc2l0aW9uID0gImJvdHRvbSIpCiNnZ3NhdmUoIi4vY29nc2NpX2ltYWdlcy90cmFpbmluZ19sb2dsaWsucG5nIixoZWlnaHQ9NSx3aWR0aD02KQpgYGAKCgpgYGB7ciBPbiBTRyBzY29yZX0KbW9kZWxfZGVsdGFzICU+JQogIG11dGF0ZSh0cmFpbl9zaXplID0gbG9nKHRyYWluX3NpemUpKSAlPiUKICBnZ3Bsb3QoYWVzKHg9dHJhaW5fc2l6ZSwgeT1zZ19zY29yZSwgY29sb3I9bW9kZWwpKSArCiAgICBnZW9tX3Ntb290aChtZXRob2Q9ImxtIiwgc2U9VCwgYWxwaGE9MC41KSArCiAgICBnZW9tX3BvaW50KHN0YXQ9ImlkZW50aXR5IiwgcG9zaXRpb249ImRvZGdlIiwgYWxwaGE9MSwgc2l6ZT0zKSArCiAgICB5bGFiKCJTRyBTQ29yZSIpICsKICAgIHhsYWIoIkxvZyBNaWxsaW9uIFRyYWluaW5nIFRva2VucyIpICsKICAgIGdndGl0bGUoIlRyYWluaW5nIFNpemUgdnMuIFN5bnRhY3RpYyBHZW5lcmFsaXphdGlvbiIpICsKICAgICNzY2FsZV9jb2xvcl9tYW51YWwodmFsdWVzID0gYygiI0E0MkVGMSIsICIjMzg5NEM4IikpICsKICAgIGZhY2V0X2dyaWQofm1vZGVsLCBzY2FsZXM9ImZyZWUiKSArCiAgICB0aGVtZShheGlzLnRleHQ9ZWxlbWVudF90ZXh0KHNpemU9MTQpLAogICAgICAgICAgc3RyaXAudGV4dC54ID0gZWxlbWVudF90ZXh0KHNpemU9MTQpLAogICAgICAgICAgbGVnZW5kLnRleHQ9ZWxlbWVudF90ZXh0KHNpemU9MTQpLAogICAgICAgICAgYXhpcy50aXRsZT1lbGVtZW50X3RleHQoc2l6ZT0xOCksCiAgICAgICAgICBsZWdlbmQucG9zaXRpb24gPSAiYm90dG9tIikKI2dnc2F2ZSgiLi9jb2dzY2lfaW1hZ2VzL3RyYWluaW5nX3NnLnBuZyIsaGVpZ2h0PTUsd2lkdGg9NikKYGBgCgojIyBTbWl0aCAmIExldnkgcmVwcm9kdWN0aW9uCgpgYGB7cn0KICAKYWxsX2RhdGEgJT4lCiAgI2ZpbHRlcihzdXJwcmlzYWwgPCAxNSwgc3VycHJpc2FsID4gMCkgJT4lCiAgZ2dwbG90KGFlcyh4PXN1cnByaXNhbCwgeT1wc3ljaG9tZXRyaWMsIGNvbG9yPXRyYWluaW5nKSkgKwogICAgc3RhdF9zbW9vdGgoc2U9VCwgYWxwaGE9MC41KSArCiAgICAjZ2VvbV9lcnJvcmJhcihjb2xvcj0iYmxhY2siLCB3aWR0aD0uMiwgcG9zaXRpb249cG9zaXRpb25fZG9kZ2Uod2lkdGg9LjkpLCBhbHBoYT0wLjMpICsKICAgICNnZW9tX3BvaW50KHN0YXQ9ImlkZW50aXR5IiwgcG9zaXRpb249ImRvZGdlIiwgYWxwaGE9MSwgc2l6ZT0zKSArCiAgICB5bGFiKCJQcm9jZXNzaW5nIFRpbWUgKG1zKSIpICsKICAgIHhsYWIoIlN1cnByaXNhbCAoYml0cykiKSArCiAgICBnZ3RpdGxlKCJTdXJwcmlzYWwgdnMuIFJlYWRpbmcgVGltZSAvIEdhemUgRHVyYXRpb24iKSArCiAgICBmYWNldF9ncmlkKGNvcnB1c35tb2RlbCwgc2NhbGVzID0gImZyZWUiKSArCiAgICAjIHNjYWxlX2NvbG9yX21hbnVhbCh2YWx1ZXMgPSBjKCJibGxpcC1sZyI9IiM0NDAxNTRGRiIsCiAgICAjICAgICAgICAgICAgICAgICAgICAgICAgICAgImJsbGlwLW1kIj0iIzM5NTY4Q0ZGIiwKICAgICMgICAgICAgICAgICAgICAgICAgICAgICAgICAiYmxsaXAtc20iPSIjMUY5NjhCRkYiLAogICAgIyAgICAgICAgICAgICAgICAgICAgICAgICAgICJibGxpcC14cyI9IiM3M0QwNTVGRiIsCiAgICAjICAgICAgICAgICAgICAgICAgICAgICAgICAgImJsbGlwLWxnLWdwdGJwZSI9IiM4ODg4ODgiLAogICAgIyAgICAgICAgICAgICAgICAgICAgICAgICAgICJibGxpcC1tZC1ncHRicGUiPSIjODg4ODg4IiwKICAgICMgICAgICAgICAgICAgICAgICAgICAgICAgICAiYmxsaXAtc20tZ3B0YnBlIj0iIzg4ODg4OCIsCiAgICAjICAgICAgICAgICAgICAgICAgICAgICAgICAgImJsbGlwLXhzLWdwdGJwZSI9IiM4ODg4ODgiKSkgKwogICAgdGhlbWUoYXhpcy50ZXh0PWVsZW1lbnRfdGV4dChzaXplPTE0KSwKICAgICAgICAgIGF4aXMudGV4dC55ID0gZWxlbWVudF90ZXh0KHNpemUgPSAxMCksCiAgICAgICAgICBzdHJpcC50ZXh0LnggPSBlbGVtZW50X3RleHQoc2l6ZT0xNCksCiAgICAgICAgICBsZWdlbmQudGV4dD1lbGVtZW50X3RleHQoc2l6ZT0xNCksCiAgICAgICAgICBheGlzLnRpdGxlPWVsZW1lbnRfdGV4dChzaXplPTE4KSwKICAgICAgICAgIGxlZ2VuZC5wb3NpdGlvbiA9ICJyaWdodCIpCmdnc2F2ZSgiLi4vaW1hZ2VzL2N1bnkyMDIwL3N1cnBfY29yci5wbmciLGhlaWdodD00LjUsd2lkdGg9MTIpCmBgYAoKCmBgYHtyfQpmdWxsX3Jlc2lkdWFscyAlPiUgI2ZpbHRlcihtb2RlbCA9PSAiZ3B0MiIsIHN0cl9kZXRlY3QodHJhaW5pbmcsICJncHRicGUiKSkgJT4lIAogIG11dGF0ZSh0cmFpbmluZ192b2NhYj1hcy5mYWN0b3IoaWZlbHNlKHN0cl9kZXRlY3QodHJhaW5pbmcsICJncHRicGUiKSwgImdwdGJwZSIsIGFzLmNoYXJhY3Rlcih0cmFpbmluZykpKSkgJT4lIAogIGdncGxvdChhZXMoeD1zdXJwcmlzYWwsIHk9cmVzaWQsIGNvbG9yPXRyYWluaW5nKSkgKwogICAgZmFjZXRfZ3JpZCh+Y29ycHVzKSArCiAgICBnZW9tX3Ntb290aCgpCmBgYAoKYGBge3J9CmZ1bGxfcmVzaWR1YWxzICU+JSByaWdodF9qb2luKGJhc2VsaW5lX3Jlc2lkdWFscywgYnk9YygiY29ycHVzIiwgImNvZGUiLCAibW9kZWwiLCAidHJhaW5pbmciLCAic2VlZCIpLCBzdWZmaXg9YygiLmZ1bGwiLCAiLmJhc2VsaW5lIikpICU+JSAKICBtdXRhdGUocmVzaWRfZGVsdGE9cmVzaWQuZnVsbCAtIHJlc2lkLmJhc2VsaW5lKSAlPiUgCiAgZ2dwbG90KGFlcyh4PXN1cnByaXNhbC5mdWxsLCB5PXJlc2lkX2RlbHRhLCBjb2xvcj10cmFpbmluZykpICsKICAgIGZhY2V0X2dyaWQofmNvcnB1cykgKwogICAgZ2VvbV9zbW9vdGgoKQpgYGA=